phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import base64
   7 import subprocess
   8 import configparser
   9
  10 from dateutil.tz import tzutc
  11 from datetime import datetime, timedelta
  12
  13 from twisted.internet import defer
  14 from twisted.python import log
  15
  16 from buildbot import locks
  17 from buildbot.data import resultspec
  18 from buildbot.changes.gitpoller import GitPoller
  19 from buildbot.config import BuilderConfig
  20 from buildbot.plugins import reporters
  21 from buildbot.plugins import schedulers
  22 from buildbot.plugins import steps
  23 from buildbot.plugins import util
  24 from buildbot.process import properties
  25 from buildbot.process import results
  26 from buildbot.process.factory import BuildFactory
  27 from buildbot.process.properties import Interpolate
  28 from buildbot.process.properties import Property
  29 from buildbot.schedulers.basic import AnyBranchScheduler
  30 from buildbot.schedulers.forcesched import BaseParameter
  31 from buildbot.schedulers.forcesched import ForceScheduler
  32 from buildbot.schedulers.forcesched import ValidationError
  33 from buildbot.steps.master import MasterShellCommand
  34 from buildbot.steps.shell import SetPropertyFromCommand
  35 from buildbot.steps.shell import ShellCommand
  36 from buildbot.steps.source.git import Git
  37 from buildbot.steps.transfer import FileDownload
  38 from buildbot.steps.transfer import FileUpload
  39 from buildbot.steps.transfer import StringDownload
  40 from buildbot.worker import Worker
  41 from buildbot.worker.local import LocalWorker
  42
  43
  44 if not os.path.exists("twistd.pid"):
  45     with open("twistd.pid", "w") as pidfile:
  46         pidfile.write("{}".format(os.getpid()))
  47
  48 # This is a sample buildmaster config file. It must be installed as
  49 # 'master.cfg' in your buildmaster's base directory.
  50
  51 ini = configparser.ConfigParser()
  52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
  53
  54 if "general" not in ini or "phase1" not in ini:
  55     raise ValueError("Fix your configuration")
  56
  57 inip1 = ini["phase1"]
  58
  59 # Globals
  60 work_dir = os.path.abspath(ini["general"].get("workdir", "."))
  61 scripts_dir = os.path.abspath("../scripts")
  62
  63 repo_url = ini["repo"].get("url")
  64
  65 rsync_defopts = ["-v", "--timeout=120"]
  66
  67 # if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
  68 #       rsync_bin_defopts += ["--contimeout=20"]
  69
  70 branches = {}
  71
  72
  73 def ini_parse_branch(section):
  74     b = {}
  75     name = section.get("name")
  76
  77     if not name:
  78         raise ValueError("missing 'name' in " + repr(section))
  79     if name in branches:
  80         raise ValueError("duplicate branch name in " + repr(section))
  81
  82     b["name"] = name
  83     b["bin_url"] = section.get("binary_url")
  84     b["bin_key"] = section.get("binary_password")
  85
  86     b["src_url"] = section.get("source_url")
  87     b["src_key"] = section.get("source_password")
  88
  89     b["gpg_key"] = section.get("gpg_key")
  90
  91     b["usign_key"] = section.get("usign_key")
  92     usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
  93     b["usign_comment"] = section.get("usign_comment", usign_comment)
  94
  95     b["config_seed"] = section.get("config_seed")
  96
  97     b["kmod_archive"] = section.getboolean("kmod_archive", False)
  98
  99     branches[name] = b
 100     log.msg("Configured branch: {}".format(name))
 101
 102
 103 # PB port can be either a numeric port or a connection string
 104 pb_port = inip1.get("port") or 9989
 105
 106 # This is the dictionary that the buildmaster pays attention to. We also use
 107 # a shorter alias to save typing.
 108 c = BuildmasterConfig = {}
 109
 110 ####### PROJECT IDENTITY
 111
 112 # the 'title' string will appear at the top of this buildbot
 113 # installation's html.WebStatus home page (linked to the
 114 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 115
 116 c["title"] = ini["general"].get("title")
 117 c["titleURL"] = ini["general"].get("title_url")
 118
 119 # the 'buildbotURL' string should point to the location where the buildbot's
 120 # internal web server (usually the html.WebStatus page) is visible. This
 121 # typically uses the port number set in the Waterfall 'status' entry, but
 122 # with an externally-visible host name which the buildbot cannot figure out
 123 # without some help.
 124
 125 c["buildbotURL"] = inip1.get("buildbot_url")
 126
 127 ####### BUILDWORKERS
 128
 129 # The 'workers' list defines the set of recognized buildworkers. Each element is
 130 # a Worker object, specifying a unique worker name and password.  The same
 131 # worker name and password must be configured on the worker.
 132
 133 c["workers"] = []
 134 NetLocks = dict()
 135
 136
 137 def ini_parse_workers(section):
 138     name = section.get("name")
 139     password = section.get("password")
 140     phase = section.getint("phase")
 141     tagonly = section.getboolean("tag_only")
 142     rsyncipv4 = section.getboolean("rsync_ipv4")
 143
 144     if not name or not password or not phase == 1:
 145         log.msg("invalid worker configuration ignored: {}".format(repr(section)))
 146         return
 147
 148     sl_props = {"tag_only": tagonly}
 149     if "dl_lock" in section:
 150         lockname = section.get("dl_lock")
 151         sl_props["dl_lock"] = lockname
 152         if lockname not in NetLocks:
 153             NetLocks[lockname] = locks.MasterLock(lockname)
 154     if "ul_lock" in section:
 155         lockname = section.get("ul_lock")
 156         sl_props["ul_lock"] = lockname
 157         if lockname not in NetLocks:
 158             NetLocks[lockname] = locks.MasterLock(lockname)
 159     if rsyncipv4:
 160         sl_props[
 161             "rsync_ipv4"
 162         ] = True  # only set prop if required, we use '+' Interpolate substitution
 163
 164     log.msg("Configured worker: {}".format(name))
 165     # NB: phase1 build factory requires workers to be single-build only
 166     c["workers"].append(Worker(name, password, max_builds=1, properties=sl_props))
 167
 168
 169 for section in ini.sections():
 170     if section.startswith("branch "):
 171         ini_parse_branch(ini[section])
 172
 173     if section.startswith("worker "):
 174         ini_parse_workers(ini[section])
 175
 176 # list of branches in build-priority order
 177 branchNames = [branches[b]["name"] for b in branches]
 178
 179 c["protocols"] = {"pb": {"port": pb_port}}
 180
 181 # coalesce builds
 182 c["collapseRequests"] = True
 183
 184 # Reduce amount of backlog data
 185 c["configurators"] = [
 186     util.JanitorConfigurator(
 187         logHorizon=timedelta(days=3),
 188         hour=6,
 189     )
 190 ]
 191
 192
 193 @defer.inlineCallbacks
 194 def getNewestCompleteTime(bldr):
 195     """Returns the complete_at of the latest completed and not SKIPPED
 196     build request for this builder, or None if there are no such build
 197     requests. We need to filter out SKIPPED requests because we're
 198     using collapseRequests=True which is unfortunately marking all
 199     previous requests as complete when new buildset is created.
 200
 201     @returns: datetime instance or None, via Deferred
 202     """
 203
 204     bldrid = yield bldr.getBuilderId()
 205     completed = yield bldr.master.data.get(
 206         ("builders", bldrid, "buildrequests"),
 207         [
 208             resultspec.Filter("complete", "eq", [True]),
 209             resultspec.Filter("results", "ne", [results.SKIPPED]),
 210         ],
 211         order=["-complete_at"],
 212         limit=1,
 213     )
 214     if not completed:
 215         return
 216
 217     complete_at = completed[0]["complete_at"]
 218
 219     last_build = yield bldr.master.data.get(
 220         ("builds",),
 221         [
 222             resultspec.Filter("builderid", "eq", [bldrid]),
 223         ],
 224         order=["-started_at"],
 225         limit=1,
 226     )
 227
 228     if last_build and last_build[0]:
 229         last_complete_at = last_build[0]["complete_at"]
 230         if last_complete_at and (last_complete_at > complete_at):
 231             return last_complete_at
 232
 233     return complete_at
 234
 235
 236 @defer.inlineCallbacks
 237 def prioritizeBuilders(master, builders):
 238     """Returns sorted list of builders by their last timestamp of completed and
 239     not skipped build, ordered first by branch name.
 240
 241     @returns: list of sorted builders
 242     """
 243
 244     bldrNamePrio = {"__Janitor": 0, "00_force_build": 0}
 245     i = 1
 246     for bname in branchNames:
 247         bldrNamePrio[bname] = i
 248         i += 1
 249
 250     def is_building(bldr):
 251         return bool(bldr.building) or bool(bldr.old_building)
 252
 253     def bldr_info(bldr):
 254         d = defer.maybeDeferred(getNewestCompleteTime, bldr)
 255         d.addCallback(lambda complete_at: (complete_at, bldr))
 256         return d
 257
 258     def bldr_sort(item):
 259         (complete_at, bldr) = item
 260
 261         pos = 99
 262         for name, prio in bldrNamePrio.items():
 263             if bldr.name.startswith(name):
 264                 pos = prio
 265                 break
 266
 267         if not complete_at:
 268             date = datetime.min
 269             complete_at = date.replace(tzinfo=tzutc())
 270
 271         if is_building(bldr):
 272             date = datetime.max
 273             complete_at = date.replace(tzinfo=tzutc())
 274
 275         return (pos, complete_at, bldr.name)
 276
 277     results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
 278     results.sort(key=bldr_sort)
 279
 280     # for r in results:
 281     #   log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
 282
 283     return [r[1] for r in results]
 284
 285
 286 c["prioritizeBuilders"] = prioritizeBuilders
 287
 288 ####### CHANGESOURCES
 289
 290 # find targets
 291 targets = set()
 292
 293
 294 def populateTargets():
 295     """fetch a shallow clone of each configured branch in turn:
 296     execute dump-target-info.pl and collate the results to ensure
 297     targets that only exist in specific branches get built.
 298     This takes a while during master startup but is executed only once.
 299     """
 300     log.msg("Populating targets, this will take time")
 301     sourcegit = work_dir + "/source.git"
 302     for branch in branchNames:
 303         if os.path.isdir(sourcegit):
 304             subprocess.call(["rm", "-rf", sourcegit])
 305
 306         subprocess.call(
 307             [
 308                 "git",
 309                 "clone",
 310                 "-q",
 311                 "--depth=1",
 312                 "--branch=" + branch,
 313                 repo_url,
 314                 sourcegit,
 315             ]
 316         )
 317
 318         os.makedirs(sourcegit + "/tmp", exist_ok=True)
 319         findtargets = subprocess.Popen(
 320             ["./scripts/dump-target-info.pl", "targets"],
 321             stdout=subprocess.PIPE,
 322             stderr=subprocess.DEVNULL,
 323             cwd=sourcegit,
 324         )
 325
 326         while True:
 327             line = findtargets.stdout.readline()
 328             if not line:
 329                 break
 330             ta = line.decode().strip().split(" ")
 331             targets.add(ta[0])
 332
 333         subprocess.call(["rm", "-rf", sourcegit])
 334
 335
 336 populateTargets()
 337
 338 # the 'change_source' setting tells the buildmaster how it should find out
 339 # about source code changes.
 340
 341 c["change_source"] = []
 342 c["change_source"].append(
 343     GitPoller(
 344         repo_url,
 345         workdir=work_dir + "/work.git",
 346         branches=branchNames,
 347         pollAtLaunch=True,
 348         pollinterval=300,
 349     )
 350 )
 351
 352 ####### SCHEDULERS
 353
 354 # Configure the Schedulers, which decide how to react to incoming changes.
 355
 356
 357 # Selector for known valid tags
 358 class TagChoiceParameter(BaseParameter):
 359     spec_attributes = ["strict", "choices"]
 360     type = "list"
 361     strict = True
 362
 363     def __init__(self, name, label=None, **kw):
 364         super().__init__(name, label, **kw)
 365         self._choice_list = []
 366
 367     def getRevTags(self, findtag=None):
 368         taglist = []
 369         branchvers = []
 370
 371         # we will filter out tags that do no match the configured branches
 372         for b in branchNames:
 373             basever = re.search(r"-([0-9]+\.[0-9]+)$", b)
 374             if basever:
 375                 branchvers.append(basever[1])
 376
 377         # grab tags from remote repository
 378         alltags = subprocess.Popen(
 379             ["git", "ls-remote", "--tags", repo_url], stdout=subprocess.PIPE
 380         )
 381
 382         while True:
 383             line = alltags.stdout.readline()
 384
 385             if not line:
 386                 break
 387
 388             (rev, tag) = line.split()
 389
 390             # does it match known format? ('vNN.NN.NN(-rcN)')
 391             tagver = re.search(
 392                 r"\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$",
 393                 tag.decode().strip(),
 394             )
 395
 396             # only list valid tags matching configured branches
 397             if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
 398                 # if we want a specific tag, ignore all that don't match
 399                 if findtag and findtag != tagver[1]:
 400                     continue
 401                 taglist.append({"rev": rev.decode().strip(), "tag": tagver[1]})
 402
 403         return taglist
 404
 405     @property
 406     def choices(self):
 407         taglist = [rt["tag"] for rt in self.getRevTags()]
 408         taglist.sort(
 409             reverse=True,
 410             key=lambda tag: tag if re.search(r"-rc[0-9]+$", tag) else tag + "-z",
 411         )
 412         taglist.insert(0, "")
 413
 414         self._choice_list = taglist
 415
 416         return self._choice_list
 417
 418     def updateFromKwargs(self, properties, kwargs, **unused):
 419         tag = self.getFromKwargs(kwargs)
 420         properties[self.name] = tag
 421
 422         # find the commit matching the tag
 423         findtag = self.getRevTags(tag)
 424
 425         if not findtag:
 426             raise ValidationError("Couldn't find tag")
 427
 428         properties["force_revision"] = findtag[0]["rev"]
 429
 430         # find the branch matching the tag
 431         branch = None
 432         branchver = re.search(r"v([0-9]+\.[0-9]+)", tag)
 433         for b in branchNames:
 434             if b.endswith(branchver[1]):
 435                 branch = b
 436
 437         if not branch:
 438             raise ValidationError("Couldn't find branch")
 439
 440         properties["force_branch"] = branch
 441
 442     def parse_from_arg(self, s):
 443         if self.strict and s not in self._choice_list:
 444             raise ValidationError(
 445                 "'%s' does not belong to list of available choices '%s'"
 446                 % (s, self._choice_list)
 447             )
 448         return s
 449
 450
 451 @util.renderer
 452 @defer.inlineCallbacks
 453 def builderNames(props):
 454     """since we have per branch and per target builders,
 455     address the relevant builder for each new buildrequest
 456     based on the request's desired branch and target.
 457     """
 458     branch = props.getProperty("branch")
 459     target = props.getProperty("target", "")
 460
 461     if target == "all":
 462         target = ""
 463
 464     # if that didn't work, try sourcestamp to find a branch
 465     if not branch:
 466         # match builders with target branch
 467         ss = props.sourcestamps[0]
 468         if ss:
 469             branch = ss["branch"]
 470         else:
 471             log.msg("couldn't find builder")
 472             return []  # nothing works
 473
 474     bname = branch + "_" + target
 475     builders = []
 476
 477     for b in (yield props.master.data.get(("builders",))):
 478         if not b["name"].startswith(bname):
 479             continue
 480         builders.append(b["name"])
 481
 482     return builders
 483
 484
 485 c["schedulers"] = []
 486 c["schedulers"].append(
 487     AnyBranchScheduler(
 488         name="all",
 489         change_filter=util.ChangeFilter(branch=branchNames),
 490         treeStableTimer=15 * 60,
 491         builderNames=builderNames,
 492     )
 493 )
 494
 495 c["schedulers"].append(
 496     ForceScheduler(
 497         name="force",
 498         buttonName="Force builds",
 499         label="Force build details",
 500         builderNames=["00_force_build"],
 501         codebases=[
 502             util.CodebaseParameter(
 503                 "",
 504                 label="Repository",
 505                 branch=util.FixedParameter(name="branch", default=""),
 506                 revision=util.FixedParameter(name="revision", default=""),
 507                 repository=util.FixedParameter(name="repository", default=""),
 508                 project=util.FixedParameter(name="project", default=""),
 509             )
 510         ],
 511         reason=util.StringParameter(
 512             name="reason",
 513             label="Reason",
 514             default="Trigger build",
 515             required=True,
 516             size=80,
 517         ),
 518         properties=[
 519             # NB: avoid nesting to simplify processing of properties
 520             util.ChoiceStringParameter(
 521                 name="target",
 522                 label="Build target",
 523                 default="all",
 524                 choices=["all"] + list(targets),
 525             ),
 526             TagChoiceParameter(name="tag", label="Build tag", default=""),
 527         ],
 528     )
 529 )
 530
 531 c["schedulers"].append(
 532     schedulers.Triggerable(name="trigger", builderNames=builderNames)
 533 )
 534
 535 ####### BUILDERS
 536
 537 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 538 # what steps, and which workers can execute them.  Note that any particular build will
 539 # only take place on one worker.
 540
 541
 542 def IsNoMasterBuild(step):
 543     return step.getProperty("branch") != "master"
 544
 545
 546 def IsUsignEnabled(step):
 547     branch = step.getProperty("branch")
 548     return branch and branches[branch].get("usign_key")
 549
 550
 551 def IsSignEnabled(step):
 552     branch = step.getProperty("branch")
 553     return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
 554
 555
 556 def IsKmodArchiveEnabled(step):
 557     branch = step.getProperty("branch")
 558     return branch and branches[branch].get("kmod_archive")
 559
 560
 561 def IsKmodArchiveAndRsyncEnabled(step):
 562     branch = step.getProperty("branch")
 563     return bool(IsKmodArchiveEnabled(step) and branches[branch].get("bin_url"))
 564
 565
 566 def GetBaseVersion(branch):
 567     if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
 568         return branch.split("-")[1]
 569     else:
 570         return "master"
 571
 572
 573 @properties.renderer
 574 def GetVersionPrefix(props):
 575     branch = props.getProperty("branch")
 576     basever = GetBaseVersion(branch)
 577     if props.hasProperty("tag") and re.match(
 578         r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]
 579     ):
 580         return "%s/" % props["tag"][1:]
 581     elif basever != "master":
 582         return "%s-SNAPSHOT/" % basever
 583     else:
 584         return ""
 585
 586
 587 @util.renderer
 588 def GetConfigSeed(props):
 589     branch = props.getProperty("branch")
 590     return branch and branches[branch].get("config_seed") or ""
 591
 592
 593 @util.renderer
 594 def GetRsyncParams(props, srcorbin, urlorkey):
 595     # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
 596     branch = props.getProperty("branch")
 597     opt = srcorbin + "_" + urlorkey
 598     return branch and branches[branch].get(opt)
 599
 600
 601 @util.renderer
 602 def GetUsignKey(props):
 603     branch = props.getProperty("branch")
 604     return branch and branches[branch].get("usign_key")
 605
 606
 607 def GetNextBuild(builder, requests):
 608     for r in requests:
 609         if r.properties:
 610             # order tagged build first
 611             if r.properties.hasProperty("tag"):
 612                 return r
 613
 614     r = requests[0]
 615     # log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
 616     return r
 617
 618
 619 def MakeEnv(overrides=None, tryccache=False):
 620     env = {
 621         "CCC": Interpolate("%(prop:cc_command:-gcc)s"),
 622         "CCXX": Interpolate("%(prop:cxx_command:-g++)s"),
 623     }
 624     if tryccache:
 625         env["CC"] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
 626         env["CXX"] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
 627         env["CCACHE"] = Interpolate("%(prop:ccache_command:-)s")
 628     else:
 629         env["CC"] = env["CCC"]
 630         env["CXX"] = env["CCXX"]
 631         env["CCACHE"] = ""
 632     if overrides is not None:
 633         env.update(overrides)
 634     return env
 635
 636
 637 @properties.renderer
 638 def NetLockDl(props, extralock=None):
 639     lock = None
 640     if props.hasProperty("dl_lock"):
 641         lock = NetLocks[props["dl_lock"]]
 642     if lock is not None:
 643         return [lock.access("exclusive")]
 644     else:
 645         return []
 646
 647
 648 @properties.renderer
 649 def NetLockUl(props):
 650     lock = None
 651     if props.hasProperty("ul_lock"):
 652         lock = NetLocks[props["ul_lock"]]
 653     if lock is not None:
 654         return [lock.access("exclusive")]
 655     else:
 656         return []
 657
 658
 659 def IsTargetSelected(target):
 660     def CheckTargetProperty(step):
 661         selected_target = step.getProperty("target", "all")
 662         if selected_target != "all" and selected_target != target:
 663             return False
 664         return True
 665
 666     return CheckTargetProperty
 667
 668
 669 @util.renderer
 670 def UsignSec2Pub(props):
 671     branch = props.getProperty("branch")
 672     try:
 673         comment = (
 674             branches[branch].get("usign_comment") or "untrusted comment: secret key"
 675         )
 676         seckey = branches[branch].get("usign_key")
 677         seckey = base64.b64decode(seckey)
 678     except Exception:
 679         return None
 680
 681     return "{}\n{}".format(
 682         re.sub(r"\bsecret key$", "public key", comment),
 683         base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]),
 684     )
 685
 686
 687 def canStartBuild(builder, wfb, request):
 688     """filter out non tag requests for tag_only workers."""
 689     wtagonly = wfb.worker.properties.getProperty("tag_only")
 690     tag = request.properties.getProperty("tag")
 691
 692     if wtagonly and not tag:
 693         return False
 694
 695     return True
 696
 697
 698 c["builders"] = []
 699
 700 workerNames = []
 701
 702 for worker in c["workers"]:
 703     workerNames.append(worker.workername)
 704
 705 # add a single LocalWorker to handle the forcebuild builder
 706 c["workers"].append(LocalWorker("__local_force_build", max_builds=1))
 707
 708 force_factory = BuildFactory()
 709 force_factory.addStep(
 710     steps.Trigger(
 711         name="trigger_build",
 712         schedulerNames=["trigger"],
 713         sourceStamps=[
 714             {
 715                 "codebase": "",
 716                 "branch": Property("force_branch"),
 717                 "revision": Property("force_revision"),
 718                 "repository": repo_url,
 719                 "project": "",
 720             }
 721         ],
 722         set_properties={
 723             "reason": Property("reason"),
 724             "tag": Property("tag"),
 725             "target": Property("target"),
 726         },
 727     )
 728 )
 729
 730 c["builders"].append(
 731     BuilderConfig(
 732         name="00_force_build", workername="__local_force_build", factory=force_factory
 733     )
 734 )
 735
 736
 737 # NB the phase1 build factory assumes workers are single-build only
 738 for target in targets:
 739     ts = target.split("/")
 740
 741     factory = BuildFactory()
 742
 743     # setup shared work directory if required
 744     factory.addStep(
 745         ShellCommand(
 746             name="sharedwd",
 747             descriptionDone="Shared work directory set up",
 748             command='test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
 749             workdir=".",
 750             haltOnFailure=True,
 751         )
 752     )
 753
 754     # find number of cores
 755     factory.addStep(
 756         SetPropertyFromCommand(
 757             name="nproc",
 758             property="nproc",
 759             description="Finding number of CPUs",
 760             command=["nproc"],
 761         )
 762     )
 763
 764     # find gcc and g++ compilers
 765     factory.addStep(
 766         FileDownload(
 767             name="dlfindbinpl",
 768             mastersrc=scripts_dir + "/findbin.pl",
 769             workerdest="../findbin.pl",
 770             mode=0o755,
 771         )
 772     )
 773
 774     factory.addStep(
 775         SetPropertyFromCommand(
 776             name="gcc",
 777             property="cc_command",
 778             description="Finding gcc command",
 779             command=["../findbin.pl", "gcc", "", ""],
 780             haltOnFailure=True,
 781         )
 782     )
 783
 784     factory.addStep(
 785         SetPropertyFromCommand(
 786             name="g++",
 787             property="cxx_command",
 788             description="Finding g++ command",
 789             command=["../findbin.pl", "g++", "", ""],
 790             haltOnFailure=True,
 791         )
 792     )
 793
 794     # see if ccache is available
 795     factory.addStep(
 796         SetPropertyFromCommand(
 797             name="ccache",
 798             property="ccache_command",
 799             description="Testing for ccache command",
 800             command=["which", "ccache"],
 801             haltOnFailure=False,
 802             flunkOnFailure=False,
 803             warnOnFailure=False,
 804             hideStepIf=lambda r, s: r == results.FAILURE,
 805         )
 806     )
 807
 808     # check out the source
 809     # Git() runs:
 810     # if repo doesn't exist: 'git clone repourl'
 811     # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
 812     # git cat-file -e <commit>
 813     # git checkout -f <commit>
 814     # git checkout -B <branch>
 815     # git rev-parse HEAD
 816     factory.addStep(
 817         Git(
 818             name="git",
 819             repourl=repo_url,
 820             mode="full",
 821             method="fresh",
 822             locks=NetLockDl,
 823             haltOnFailure=True,
 824         )
 825     )
 826
 827     # workaround for https://github.com/openwrt/buildbot/issues/5
 828     factory.addStep(
 829         Git(
 830             name="git me once more please",
 831             repourl=repo_url,
 832             mode="full",
 833             method="fresh",
 834             locks=NetLockDl,
 835             haltOnFailure=True,
 836         )
 837     )
 838
 839     # update remote refs
 840     factory.addStep(
 841         ShellCommand(
 842             name="fetchrefs",
 843             description="Fetching Git remote refs",
 844             descriptionDone="Git remote refs fetched",
 845             command=[
 846                 "git",
 847                 "fetch",
 848                 "origin",
 849                 Interpolate(
 850                     "+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s"
 851                 ),
 852             ],
 853             haltOnFailure=True,
 854         )
 855     )
 856
 857     # getver.sh requires local branches to track upstream otherwise version computation fails.
 858     # Git() does not set tracking branches when cloning or switching, so work around this here
 859     factory.addStep(
 860         ShellCommand(
 861             name="trackupstream",
 862             description="Setting upstream branch",
 863             descriptionDone="getver.sh is happy now",
 864             command=["git", "branch", "-u", Interpolate("origin/%(prop:branch)s")],
 865             haltOnFailure=True,
 866         )
 867     )
 868
 869     # Verify that Git HEAD points to a tag or branch
 870     # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
 871     factory.addStep(
 872         ShellCommand(
 873             name="gitverify",
 874             description="Ensuring that Git HEAD is pointing to a branch or tag",
 875             descriptionDone="Git HEAD is sane",
 876             command='git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
 877             haltOnFailure=True,
 878         )
 879     )
 880
 881     factory.addStep(
 882         StringDownload(
 883             name="ccachecc",
 884             s='#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
 885             workerdest="../ccache_cc.sh",
 886             mode=0o755,
 887         )
 888     )
 889
 890     factory.addStep(
 891         StringDownload(
 892             name="ccachecxx",
 893             s='#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
 894             workerdest="../ccache_cxx.sh",
 895             mode=0o755,
 896         )
 897     )
 898
 899     # feed
 900     factory.addStep(
 901         ShellCommand(
 902             name="updatefeeds",
 903             description="Updating feeds",
 904             command=["./scripts/feeds", "update"],
 905             env=MakeEnv(tryccache=True),
 906             haltOnFailure=True,
 907             locks=NetLockDl,
 908         )
 909     )
 910
 911     # feed
 912     factory.addStep(
 913         ShellCommand(
 914             name="installfeeds",
 915             description="Installing feeds",
 916             command=["./scripts/feeds", "install", "-a"],
 917             env=MakeEnv(tryccache=True),
 918             haltOnFailure=True,
 919         )
 920     )
 921
 922     # seed config
 923     factory.addStep(
 924         StringDownload(
 925             name="dlconfigseed",
 926             s=Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
 927             workerdest=".config",
 928             mode=0o644,
 929         )
 930     )
 931
 932     # configure
 933     factory.addStep(
 934         ShellCommand(
 935             name="newconfig",
 936             descriptionDone=".config seeded",
 937             command=Interpolate(
 938                 "printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config",
 939                 target=ts[0],
 940                 subtarget=ts[1],
 941                 usign=GetUsignKey,
 942             ),
 943         )
 944     )
 945
 946     factory.addStep(
 947         ShellCommand(
 948             name="defconfig",
 949             description="Populating .config",
 950             command=["make", "defconfig"],
 951             env=MakeEnv(),
 952         )
 953     )
 954
 955     # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
 956     factory.addStep(
 957         ShellCommand(
 958             name="checkarch",
 959             description="Checking architecture",
 960             descriptionDone="Architecture validated",
 961             command='grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config'
 962             % (ts[0], ts[1]),
 963             logEnviron=False,
 964             want_stdout=False,
 965             want_stderr=False,
 966             haltOnFailure=True,
 967             flunkOnFailure=False,  # this is not a build FAILURE - TODO mark build as SKIPPED
 968         )
 969     )
 970
 971     # find libc suffix
 972     factory.addStep(
 973         SetPropertyFromCommand(
 974             name="libc",
 975             property="libc",
 976             description="Finding libc suffix",
 977             command=[
 978                 "sed",
 979                 "-ne",
 980                 '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }',
 981                 ".config",
 982             ],
 983         )
 984     )
 985
 986     # install build key
 987     factory.addStep(
 988         StringDownload(
 989             name="dlkeybuildpub",
 990             s=Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
 991             workerdest="key-build.pub",
 992             mode=0o600,
 993             doStepIf=IsUsignEnabled,
 994         )
 995     )
 996
 997     factory.addStep(
 998         StringDownload(
 999             name="dlkeybuild",
1000             s="# fake private key",
1001             workerdest="key-build",
1002             mode=0o600,
1003             doStepIf=IsUsignEnabled,
1004         )
1005     )
1006
1007     factory.addStep(
1008         StringDownload(
1009             name="dlkeybuilducert",
1010             s="# fake certificate",
1011             workerdest="key-build.ucert",
1012             mode=0o600,
1013             doStepIf=IsUsignEnabled,
1014         )
1015     )
1016
1017     # prepare dl
1018     factory.addStep(
1019         ShellCommand(
1020             name="dldir",
1021             description="Preparing dl/",
1022             descriptionDone="dl/ prepared",
1023             command='mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
1024             workdir=Property("builddir"),
1025             logEnviron=False,
1026             want_stdout=False,
1027         )
1028     )
1029
1030     # cleanup dl
1031     factory.addStep(
1032         ShellCommand(
1033             name="dlprune",
1034             description="Pruning dl/",
1035             descriptionDone="dl/ pruned",
1036             command="find dl/ -mindepth 1 -atime +15 -delete -print",
1037             logEnviron=False,
1038         )
1039     )
1040
1041     # prepare tar
1042     factory.addStep(
1043         ShellCommand(
1044             name="dltar",
1045             description="Building and installing GNU tar",
1046             descriptionDone="GNU tar built and installed",
1047             command=[
1048                 "make",
1049                 Interpolate("-j%(prop:nproc:-1)s"),
1050                 "tools/tar/compile",
1051                 "V=s",
1052             ],
1053             env=MakeEnv(tryccache=True),
1054             haltOnFailure=True,
1055         )
1056     )
1057
1058     # populate dl
1059     factory.addStep(
1060         ShellCommand(
1061             name="dlrun",
1062             description="Populating dl/",
1063             descriptionDone="dl/ populated",
1064             command=["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
1065             env=MakeEnv(),
1066             logEnviron=False,
1067             locks=NetLockDl,
1068         )
1069     )
1070
1071     factory.addStep(
1072         ShellCommand(
1073             name="cleanbase",
1074             description="Cleaning base-files",
1075             command=["make", "package/base-files/clean", "V=s"],
1076         )
1077     )
1078
1079     # build
1080     factory.addStep(
1081         ShellCommand(
1082             name="tools",
1083             description="Building and installing tools",
1084             descriptionDone="Tools built and installed",
1085             command=[
1086                 "make",
1087                 Interpolate("-j%(prop:nproc:-1)s"),
1088                 "tools/install",
1089                 "V=s",
1090             ],
1091             env=MakeEnv(tryccache=True),
1092             haltOnFailure=True,
1093         )
1094     )
1095
1096     factory.addStep(
1097         ShellCommand(
1098             name="toolchain",
1099             description="Building and installing toolchain",
1100             descriptionDone="Toolchain built and installed",
1101             command=[
1102                 "make",
1103                 Interpolate("-j%(prop:nproc:-1)s"),
1104                 "toolchain/install",
1105                 "V=s",
1106             ],
1107             env=MakeEnv(),
1108             haltOnFailure=True,
1109         )
1110     )
1111
1112     factory.addStep(
1113         ShellCommand(
1114             name="kmods",
1115             description="Building kmods",
1116             descriptionDone="Kmods built",
1117             command=[
1118                 "make",
1119                 Interpolate("-j%(prop:nproc:-1)s"),
1120                 "target/compile",
1121                 "V=s",
1122                 "IGNORE_ERRORS=n m",
1123                 "BUILD_LOG=1",
1124             ],
1125             env=MakeEnv(),
1126             haltOnFailure=True,
1127         )
1128     )
1129
1130     # find kernel version
1131     factory.addStep(
1132         SetPropertyFromCommand(
1133             name="kernelversion",
1134             property="kernelversion",
1135             description="Finding the effective Kernel version",
1136             command="make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
1137             env={"TOPDIR": Interpolate("%(prop:builddir)s/build")},
1138         )
1139     )
1140
1141     factory.addStep(
1142         ShellCommand(
1143             name="pkgclean",
1144             description="Cleaning up package build",
1145             descriptionDone="Package build cleaned up",
1146             command=["make", "package/cleanup", "V=s"],
1147         )
1148     )
1149
1150     factory.addStep(
1151         ShellCommand(
1152             name="pkgbuild",
1153             description="Building packages",
1154             descriptionDone="Packages built",
1155             command=[
1156                 "make",
1157                 Interpolate("-j%(prop:nproc:-1)s"),
1158                 "package/compile",
1159                 "V=s",
1160                 "IGNORE_ERRORS=n m",
1161                 "BUILD_LOG=1",
1162             ],
1163             env=MakeEnv(),
1164             haltOnFailure=True,
1165         )
1166     )
1167
1168     factory.addStep(
1169         ShellCommand(
1170             name="pkginstall",
1171             description="Installing packages",
1172             descriptionDone="Packages installed",
1173             command=[
1174                 "make",
1175                 Interpolate("-j%(prop:nproc:-1)s"),
1176                 "package/install",
1177                 "V=s",
1178             ],
1179             env=MakeEnv(),
1180             haltOnFailure=True,
1181         )
1182     )
1183
1184     factory.addStep(
1185         ShellCommand(
1186             name="pkgindex",
1187             description="Indexing packages",
1188             descriptionDone="Packages indexed",
1189             command=[
1190                 "make",
1191                 Interpolate("-j%(prop:nproc:-1)s"),
1192                 "package/index",
1193                 "V=s",
1194                 "CONFIG_SIGNED_PACKAGES=",
1195             ],
1196             env=MakeEnv(),
1197             haltOnFailure=True,
1198         )
1199     )
1200
1201     factory.addStep(
1202         ShellCommand(
1203             name="images",
1204             description="Building and installing images",
1205             descriptionDone="Images built and installed",
1206             command=[
1207                 "make",
1208                 Interpolate("-j%(prop:nproc:-1)s"),
1209                 "target/install",
1210                 "V=s",
1211             ],
1212             env=MakeEnv(),
1213             haltOnFailure=True,
1214         )
1215     )
1216
1217     factory.addStep(
1218         ShellCommand(
1219             name="buildinfo",
1220             description="Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
1221             command="make -j1 buildinfo V=s || true",
1222             env=MakeEnv(),
1223             haltOnFailure=True,
1224         )
1225     )
1226
1227     factory.addStep(
1228         ShellCommand(
1229             name="json_overview_image_info",
1230             description="Generating profiles.json in target folder",
1231             command="make -j1 json_overview_image_info V=s || true",
1232             env=MakeEnv(),
1233             haltOnFailure=True,
1234         )
1235     )
1236
1237     factory.addStep(
1238         ShellCommand(
1239             name="checksums",
1240             description="Calculating checksums",
1241             descriptionDone="Checksums calculated",
1242             command=["make", "-j1", "checksum", "V=s"],
1243             env=MakeEnv(),
1244             haltOnFailure=True,
1245         )
1246     )
1247
1248     factory.addStep(
1249         ShellCommand(
1250             name="kmoddir",
1251             descriptionDone="Kmod directory created",
1252             command=[
1253                 "mkdir",
1254                 "-p",
1255                 Interpolate(
1256                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s",
1257                     target=ts[0],
1258                     subtarget=ts[1],
1259                 ),
1260             ],
1261             haltOnFailure=True,
1262             doStepIf=IsKmodArchiveEnabled,
1263         )
1264     )
1265
1266     factory.addStep(
1267         ShellCommand(
1268             name="kmodprepare",
1269             description="Preparing kmod archive",
1270             descriptionDone="Kmod archive prepared",
1271             command=[
1272                 "rsync",
1273                 "--include=/kmod-*.ipk",
1274                 "--exclude=*",
1275                 "-va",
1276                 Interpolate(
1277                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/",
1278                     target=ts[0],
1279                     subtarget=ts[1],
1280                 ),
1281                 Interpolate(
1282                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1283                     target=ts[0],
1284                     subtarget=ts[1],
1285                 ),
1286             ],
1287             haltOnFailure=True,
1288             doStepIf=IsKmodArchiveEnabled,
1289         )
1290     )
1291
1292     factory.addStep(
1293         ShellCommand(
1294             name="kmodindex",
1295             description="Indexing kmod archive",
1296             descriptionDone="Kmod archive indexed",
1297             command=[
1298                 "make",
1299                 Interpolate("-j%(prop:nproc:-1)s"),
1300                 "package/index",
1301                 "V=s",
1302                 "CONFIG_SIGNED_PACKAGES=",
1303                 Interpolate(
1304                     "PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1305                     target=ts[0],
1306                     subtarget=ts[1],
1307                 ),
1308             ],
1309             env=MakeEnv(),
1310             haltOnFailure=True,
1311             doStepIf=IsKmodArchiveEnabled,
1312         )
1313     )
1314
1315     # sign
1316     factory.addStep(
1317         MasterShellCommand(
1318             name="signprepare",
1319             descriptionDone="Temporary signing directory prepared",
1320             command=["mkdir", "-p", "%s/signing" % (work_dir)],
1321             haltOnFailure=True,
1322             doStepIf=IsSignEnabled,
1323         )
1324     )
1325
1326     factory.addStep(
1327         ShellCommand(
1328             name="signpack",
1329             description="Packing files to sign",
1330             descriptionDone="Files to sign packed",
1331             command=Interpolate(
1332                 "find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz",
1333                 target=ts[0],
1334                 subtarget=ts[1],
1335             ),
1336             haltOnFailure=True,
1337             doStepIf=IsSignEnabled,
1338         )
1339     )
1340
1341     factory.addStep(
1342         FileUpload(
1343             workersrc="sign.tar.gz",
1344             masterdest="%s/signing/%s.%s.tar.gz" % (work_dir, ts[0], ts[1]),
1345             haltOnFailure=True,
1346             doStepIf=IsSignEnabled,
1347         )
1348     )
1349
1350     factory.addStep(
1351         MasterShellCommand(
1352             name="signfiles",
1353             description="Signing files",
1354             descriptionDone="Files signed",
1355             command=[
1356                 "%s/signall.sh" % (scripts_dir),
1357                 "%s/signing/%s.%s.tar.gz" % (work_dir, ts[0], ts[1]),
1358                 Interpolate("%(prop:branch)s"),
1359             ],
1360             env={"CONFIG_INI": os.getenv("BUILDMASTER_CONFIG", "./config.ini")},
1361             haltOnFailure=True,
1362             doStepIf=IsSignEnabled,
1363         )
1364     )
1365
1366     factory.addStep(
1367         FileDownload(
1368             name="dlsigntargz",
1369             mastersrc="%s/signing/%s.%s.tar.gz" % (work_dir, ts[0], ts[1]),
1370             workerdest="sign.tar.gz",
1371             haltOnFailure=True,
1372             doStepIf=IsSignEnabled,
1373         )
1374     )
1375
1376     factory.addStep(
1377         ShellCommand(
1378             name="signunpack",
1379             description="Unpacking signed files",
1380             descriptionDone="Signed files unpacked",
1381             command=["tar", "-xzf", "sign.tar.gz"],
1382             haltOnFailure=True,
1383             doStepIf=IsSignEnabled,
1384         )
1385     )
1386
1387     # upload
1388     factory.addStep(
1389         ShellCommand(
1390             name="dirprepare",
1391             descriptionDone="Upload directory structure prepared",
1392             command=[
1393                 "mkdir",
1394                 "-p",
1395                 Interpolate(
1396                     "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s",
1397                     target=ts[0],
1398                     subtarget=ts[1],
1399                     prefix=GetVersionPrefix,
1400                 ),
1401             ],
1402             haltOnFailure=True,
1403         )
1404     )
1405
1406     factory.addStep(
1407         ShellCommand(
1408             name="linkprepare",
1409             descriptionDone="Repository symlink prepared",
1410             command=[
1411                 "ln",
1412                 "-s",
1413                 "-f",
1414                 Interpolate(
1415                     "../packages-%(kw:basever)s",
1416                     basever=util.Transform(GetBaseVersion, Property("branch")),
1417                 ),
1418                 Interpolate(
1419                     "tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix
1420                 ),
1421             ],
1422             doStepIf=IsNoMasterBuild,
1423             haltOnFailure=True,
1424         )
1425     )
1426
1427     factory.addStep(
1428         ShellCommand(
1429             name="kmoddirprepare",
1430             descriptionDone="Kmod archive upload directory prepared",
1431             command=[
1432                 "mkdir",
1433                 "-p",
1434                 Interpolate(
1435                     "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s",
1436                     target=ts[0],
1437                     subtarget=ts[1],
1438                     prefix=GetVersionPrefix,
1439                 ),
1440             ],
1441             haltOnFailure=True,
1442             doStepIf=IsKmodArchiveEnabled,
1443         )
1444     )
1445
1446     factory.addStep(
1447         ShellCommand(
1448             name="dirupload",
1449             description="Uploading directory structure",
1450             descriptionDone="Directory structure uploaded",
1451             command=["rsync", Interpolate("-az%(prop:rsync_ipv4:+4)s")]
1452             + rsync_defopts
1453             + [
1454                 "tmp/upload/",
1455                 Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url")),
1456             ],
1457             env={
1458                 "RSYNC_PASSWORD": Interpolate(
1459                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1460                 )
1461             },
1462             haltOnFailure=True,
1463             logEnviron=False,
1464             locks=NetLockUl,
1465             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1466         )
1467     )
1468
1469     # download remote sha256sums to 'target-sha256sums'
1470     factory.addStep(
1471         ShellCommand(
1472             name="target-sha256sums",
1473             description="Fetching remote sha256sums for target",
1474             descriptionDone="Remote sha256sums for target fetched",
1475             command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")]
1476             + rsync_defopts
1477             + [
1478                 Interpolate(
1479                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums",
1480                     url=GetRsyncParams.withArgs("bin", "url"),
1481                     target=ts[0],
1482                     subtarget=ts[1],
1483                     prefix=GetVersionPrefix,
1484                 ),
1485                 "target-sha256sums",
1486             ],
1487             env={
1488                 "RSYNC_PASSWORD": Interpolate(
1489                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1490                 )
1491             },
1492             logEnviron=False,
1493             haltOnFailure=False,
1494             flunkOnFailure=False,
1495             warnOnFailure=False,
1496             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1497         )
1498     )
1499
1500     # build list of files to upload
1501     factory.addStep(
1502         FileDownload(
1503             name="dlsha2rsyncpl",
1504             mastersrc=scripts_dir + "/sha2rsync.pl",
1505             workerdest="../sha2rsync.pl",
1506             mode=0o755,
1507         )
1508     )
1509
1510     factory.addStep(
1511         ShellCommand(
1512             name="buildlist",
1513             description="Building list of files to upload",
1514             descriptionDone="List of files to upload built",
1515             command=[
1516                 "../sha2rsync.pl",
1517                 "target-sha256sums",
1518                 Interpolate(
1519                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
1520                     target=ts[0],
1521                     subtarget=ts[1],
1522                 ),
1523                 "rsynclist",
1524             ],
1525             haltOnFailure=True,
1526         )
1527     )
1528
1529     factory.addStep(
1530         FileDownload(
1531             name="dlrsync.sh",
1532             mastersrc=scripts_dir + "/rsync.sh",
1533             workerdest="../rsync.sh",
1534             mode=0o755,
1535         )
1536     )
1537
1538     # upload new files and update existing ones
1539     factory.addStep(
1540         ShellCommand(
1541             name="targetupload",
1542             description="Uploading target files",
1543             descriptionDone="Target files uploaded",
1544             command=[
1545                 "../rsync.sh",
1546                 "--exclude=/kmods/",
1547                 "--files-from=rsynclist",
1548                 "--delay-updates",
1549                 "--partial-dir=.~tmp~%s~%s" % (ts[0], ts[1]),
1550             ]
1551             + rsync_defopts
1552             + [
1553                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1554                 Interpolate(
1555                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
1556                     target=ts[0],
1557                     subtarget=ts[1],
1558                 ),
1559                 Interpolate(
1560                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
1561                     url=GetRsyncParams.withArgs("bin", "url"),
1562                     target=ts[0],
1563                     subtarget=ts[1],
1564                     prefix=GetVersionPrefix,
1565                 ),
1566             ],
1567             env={
1568                 "RSYNC_PASSWORD": Interpolate(
1569                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1570                 )
1571             },
1572             haltOnFailure=True,
1573             logEnviron=False,
1574             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1575         )
1576     )
1577
1578     # delete files which don't exist locally
1579     factory.addStep(
1580         ShellCommand(
1581             name="targetprune",
1582             description="Pruning target files",
1583             descriptionDone="Target files pruned",
1584             command=[
1585                 "../rsync.sh",
1586                 "--exclude=/kmods/",
1587                 "--delete",
1588                 "--existing",
1589                 "--ignore-existing",
1590                 "--delay-updates",
1591                 "--partial-dir=.~tmp~%s~%s" % (ts[0], ts[1]),
1592             ]
1593             + rsync_defopts
1594             + [
1595                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1596                 Interpolate(
1597                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
1598                     target=ts[0],
1599                     subtarget=ts[1],
1600                 ),
1601                 Interpolate(
1602                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
1603                     url=GetRsyncParams.withArgs("bin", "url"),
1604                     target=ts[0],
1605                     subtarget=ts[1],
1606                     prefix=GetVersionPrefix,
1607                 ),
1608             ],
1609             env={
1610                 "RSYNC_PASSWORD": Interpolate(
1611                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1612                 )
1613             },
1614             haltOnFailure=True,
1615             logEnviron=False,
1616             locks=NetLockUl,
1617             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1618         )
1619     )
1620
1621     factory.addStep(
1622         ShellCommand(
1623             name="kmodupload",
1624             description="Uploading kmod archive",
1625             descriptionDone="Kmod archive uploaded",
1626             command=[
1627                 "../rsync.sh",
1628                 "--delete",
1629                 "--delay-updates",
1630                 "--partial-dir=.~tmp~%s~%s" % (ts[0], ts[1]),
1631             ]
1632             + rsync_defopts
1633             + [
1634                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1635                 Interpolate(
1636                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1637                     target=ts[0],
1638                     subtarget=ts[1],
1639                 ),
1640                 Interpolate(
1641                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/",
1642                     url=GetRsyncParams.withArgs("bin", "url"),
1643                     target=ts[0],
1644                     subtarget=ts[1],
1645                     prefix=GetVersionPrefix,
1646                 ),
1647             ],
1648             env={
1649                 "RSYNC_PASSWORD": Interpolate(
1650                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1651                 )
1652             },
1653             haltOnFailure=True,
1654             logEnviron=False,
1655             locks=NetLockUl,
1656             doStepIf=IsKmodArchiveAndRsyncEnabled,
1657         )
1658     )
1659
1660     factory.addStep(
1661         ShellCommand(
1662             name="sourcelist",
1663             description="Finding source archives to upload",
1664             descriptionDone="Source archives to upload found",
1665             command="find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1666             haltOnFailure=True,
1667         )
1668     )
1669
1670     factory.addStep(
1671         ShellCommand(
1672             name="sourceupload",
1673             description="Uploading source archives",
1674             descriptionDone="Source archives uploaded",
1675             command=[
1676                 "../rsync.sh",
1677                 "--files-from=sourcelist",
1678                 "--size-only",
1679                 "--delay-updates",
1680             ]
1681             + rsync_defopts
1682             + [
1683                 Interpolate(
1684                     "--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s",
1685                     target=ts[0],
1686                     subtarget=ts[1],
1687                 ),
1688                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1689                 "dl/",
1690                 Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url")),
1691             ],
1692             env={
1693                 "RSYNC_PASSWORD": Interpolate(
1694                     "%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")
1695                 )
1696             },
1697             haltOnFailure=True,
1698             logEnviron=False,
1699             locks=NetLockUl,
1700             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1701         )
1702     )
1703
1704     factory.addStep(
1705         ShellCommand(
1706             name="df",
1707             description="Reporting disk usage",
1708             command=["df", "-h", "."],
1709             env={"LC_ALL": "C"},
1710             logEnviron=False,
1711             haltOnFailure=False,
1712             flunkOnFailure=False,
1713             warnOnFailure=False,
1714             alwaysRun=True,
1715         )
1716     )
1717
1718     factory.addStep(
1719         ShellCommand(
1720             name="du",
1721             description="Reporting estimated file space usage",
1722             command=["du", "-sh", "."],
1723             env={"LC_ALL": "C"},
1724             logEnviron=False,
1725             haltOnFailure=False,
1726             flunkOnFailure=False,
1727             warnOnFailure=False,
1728             alwaysRun=True,
1729         )
1730     )
1731
1732     factory.addStep(
1733         ShellCommand(
1734             name="ccachestat",
1735             description="Reporting ccache stats",
1736             command=["ccache", "-s"],
1737             logEnviron=False,
1738             want_stderr=False,
1739             haltOnFailure=False,
1740             flunkOnFailure=False,
1741             warnOnFailure=False,
1742             doStepIf=util.Transform(bool, Property("ccache_command")),
1743         )
1744     )
1745
1746     for brname in branchNames:
1747         bldrname = brname + "_" + target
1748         c["builders"].append(
1749             BuilderConfig(
1750                 name=bldrname,
1751                 workernames=workerNames,
1752                 factory=factory,
1753                 tags=[
1754                     brname,
1755                 ],
1756                 nextBuild=GetNextBuild,
1757                 canStartBuild=canStartBuild,
1758             )
1759         )
1760
1761
1762 ####### STATUS TARGETS
1763
1764 # 'status' is a list of Status Targets. The results of each build will be
1765 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1766 # including web pages, email senders, and IRC bots.
1767
1768 if "status_bind" in inip1:
1769     c["www"] = {
1770         "port": inip1.get("status_bind"),
1771         "plugins": {"waterfall_view": True, "console_view": True, "grid_view": True},
1772     }
1773
1774     if "status_user" in inip1 and "status_password" in inip1:
1775         c["www"]["auth"] = util.UserPasswordAuth(
1776             [(inip1.get("status_user"), inip1.get("status_password"))]
1777         )
1778         c["www"]["authz"] = util.Authz(
1779             allowRules=[util.AnyControlEndpointMatcher(role="admins")],
1780             roleMatchers=[
1781                 util.RolesFromUsername(
1782                     roles=["admins"], usernames=[inip1.get("status_user")]
1783                 )
1784             ],
1785         )
1786
1787 c["services"] = []
1788 if ini.has_section("irc"):
1789     iniirc = ini["irc"]
1790     irc_host = iniirc.get("host", None)
1791     irc_port = iniirc.getint("port", 6667)
1792     irc_chan = iniirc.get("channel", None)
1793     irc_nick = iniirc.get("nickname", None)
1794     irc_pass = iniirc.get("password", None)
1795
1796     if irc_host and irc_nick and irc_chan:
1797         irc = reporters.IRC(
1798             irc_host,
1799             irc_nick,
1800             port=irc_port,
1801             password=irc_pass,
1802             channels=[irc_chan],
1803             notify_events=["exception", "problem", "recovery"],
1804         )
1805
1806         c["services"].append(irc)
1807
1808 c["revlink"] = util.RevlinkMatch(
1809     [r"https://git.openwrt.org/openwrt/(.*).git"],
1810     r"https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s",
1811 )
1812
1813 ####### DB URL
1814
1815 c["db"] = {
1816     # This specifies what database buildbot uses to store its state.  You can leave
1817     # this at its default for all but the largest installations.
1818     "db_url": "sqlite:///state.sqlite",
1819 }
1820
1821 c["buildbotNetUsageData"] = None