phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### PROJECT IDENTITY
  22
  23 # the 'title' string will appear at the top of this buildbot
  24 # installation's html.WebStatus home page (linked to the
  25 # 'titleURL') and is embedded in the title of the waterfall HTML page.
  26
  27 c['title'] = ini.get("general", "title")
  28 c['titleURL'] = ini.get("general", "title_url")
  29
  30 # the 'buildbotURL' string should point to the location where the buildbot's
  31 # internal web server (usually the html.WebStatus page) is visible. This
  32 # typically uses the port number set in the Waterfall 'status' entry, but
  33 # with an externally-visible host name which the buildbot cannot figure out
  34 # without some help.
  35
  36 c['buildbotURL'] = ini.get("general", "buildbot_url")
  37
  38 ####### BUILDSLAVES
  39
  40 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  41 # a BuildSlave object, specifying a unique slave name and password.  The same
  42 # slave name and password must be configured on the slave.
  43 from buildbot.buildslave import BuildSlave
  44
  45 slave_port = 9989
  46
  47 if ini.has_option("general", "port"):
  48         slave_port = ini.getint("general", "port")
  49
  50 c['slaves'] = []
  51 NetLocks = dict()
  52
  53 for section in ini.sections():
  54         if section.startswith("slave "):
  55                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  56                         sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'max_builds':1, 'shared_wd':False }
  57                         name = ini.get(section, "name")
  58                         password = ini.get(section, "password")
  59                         max_builds = 1
  60                         if ini.has_option(section, "builds"):
  61                                 max_builds = ini.getint(section, "builds")
  62                                 sl_props['max_builds'] = max_builds
  63                         if ini.has_option(section, "cleanup"):
  64                                 sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
  65                         if ini.has_option(section, "dl_lock"):
  66                                 lockname = ini.get(section, "dl_lock")
  67                                 sl_props['dl_lock'] = lockname
  68                                 if lockname not in NetLocks:
  69                                         NetLocks[lockname] = locks.MasterLock(lockname)
  70                         if ini.has_option(section, "ul_lock"):
  71                                 lockname = ini.get(section, "dl_lock")
  72                                 sl_props['ul_lock'] = lockname
  73                                 if lockname not in NetLocks:
  74                                         NetLocks[lockname] = locks.MasterLock(lockname)
  75                         if ini.has_option(section, "shared_wd"):
  76                                 shared_wd = ini.getboolean(section, "shared_wd")
  77                                 sl_props['shared_wd'] = shared_wd
  78                                 if shared_wd and (max_builds != 1):
  79                                         raise ValueError('max_builds must be 1 with shared workdir!')
  80                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds, properties = sl_props))
  81
  82 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  83 # This must match the value configured into the buildslaves (with their
  84 # --master option)
  85 c['slavePortnum'] = slave_port
  86
  87 # coalesce builds
  88 c['mergeRequests'] = True
  89
  90 # Reduce amount of backlog data
  91 c['buildHorizon'] = 30
  92 c['logHorizon'] = 20
  93
  94 ####### CHANGESOURCES
  95
  96 home_dir = os.path.abspath(ini.get("general", "homedir"))
  97 tree_expire = 0
  98 other_builds = 0
  99 cc_version = None
 100
 101 cc_command = "gcc"
 102 cxx_command = "g++"
 103
 104 if ini.has_option("general", "expire"):
 105         tree_expire = ini.getint("general", "expire")
 106
 107 if ini.has_option("general", "other_builds"):
 108         other_builds = ini.getint("general", "other_builds")
 109
 110 if ini.has_option("general", "cc_version"):
 111         cc_version = ini.get("general", "cc_version").split()
 112         if len(cc_version) == 1:
 113                 cc_version = ["eq", cc_version[0]]
 114
 115 repo_url = ini.get("repo", "url")
 116 repo_branch = "master"
 117
 118 if ini.has_option("repo", "branch"):
 119         repo_branch = ini.get("repo", "branch")
 120
 121 rsync_bin_url = ini.get("rsync", "binary_url")
 122 rsync_bin_key = ini.get("rsync", "binary_password")
 123
 124 rsync_src_url = None
 125 rsync_src_key = None
 126
 127 if ini.has_option("rsync", "source_url"):
 128         rsync_src_url = ini.get("rsync", "source_url")
 129         rsync_src_key = ini.get("rsync", "source_password")
 130
 131 gpg_home = "~/.gnupg"
 132 gpg_keyid = None
 133 gpg_comment = "Unattended build signature"
 134 gpg_passfile = "/dev/null"
 135
 136 if ini.has_option("gpg", "home"):
 137         gpg_home = ini.get("gpg", "home")
 138
 139 if ini.has_option("gpg", "keyid"):
 140         gpg_keyid = ini.get("gpg", "keyid")
 141
 142 if ini.has_option("gpg", "comment"):
 143         gpg_comment = ini.get("gpg", "comment")
 144
 145 if ini.has_option("gpg", "passfile"):
 146         gpg_passfile = ini.get("gpg", "passfile")
 147
 148 enable_kmod_archive = True
 149
 150
 151 # find targets
 152 targets = [ ]
 153
 154 if not os.path.isdir(home_dir+'/source.git'):
 155         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 156 else:
 157         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 158
 159 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 160         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 161
 162 while True:
 163         line = findtargets.stdout.readline()
 164         if not line:
 165                 break
 166         ta = line.strip().split(' ')
 167         targets.append(ta[0])
 168
 169
 170 # the 'change_source' setting tells the buildmaster how it should find out
 171 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 172
 173 from buildbot.changes.gitpoller import GitPoller
 174 c['change_source'] = []
 175 c['change_source'].append(GitPoller(
 176         repo_url,
 177         workdir=home_dir+'/work.git', branch=repo_branch,
 178         pollinterval=300))
 179
 180 ####### SCHEDULERS
 181
 182 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 183 # case, just kick off a 'basebuild' build
 184
 185 from buildbot.schedulers.basic import SingleBranchScheduler
 186 from buildbot.schedulers.forcesched import ForceScheduler
 187 from buildbot.changes import filter
 188 c['schedulers'] = []
 189 c['schedulers'].append(SingleBranchScheduler(
 190         name="all",
 191         change_filter=filter.ChangeFilter(branch=repo_branch),
 192         treeStableTimer=60,
 193         builderNames=targets))
 194
 195 c['schedulers'].append(ForceScheduler(
 196         name="force",
 197         builderNames=targets))
 198
 199 ####### BUILDERS
 200
 201 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 202 # what steps, and which slaves can execute them.  Note that any particular build will
 203 # only take place on one slave.
 204
 205 from buildbot.process.factory import BuildFactory
 206 from buildbot.steps.source.git import Git
 207 from buildbot.steps.shell import ShellCommand
 208 from buildbot.steps.shell import SetPropertyFromCommand
 209 from buildbot.steps.transfer import FileUpload
 210 from buildbot.steps.transfer import FileDownload
 211 from buildbot.steps.master import MasterShellCommand
 212 from buildbot.process.properties import Interpolate
 213 from buildbot.process import properties
 214
 215
 216 CleanTargetMap = [
 217         [ "tools",      "tools/clean"                   ],
 218         [ "chain",      "toolchain/clean"               ],
 219         [ "linux",      "target/linux/clean"    ],
 220         [ "dir",        "dirclean"                              ],
 221         [ "dist",       "distclean"                             ]
 222 ]
 223
 224 def IsMakeCleanRequested(pattern):
 225         def CheckCleanProperty(step):
 226                 val = step.getProperty("clean")
 227                 if val and re.match(pattern, val):
 228                         return True
 229                 else:
 230                         return False
 231
 232         return CheckCleanProperty
 233
 234 def IsCleanupRequested(step):
 235         shared_wd = step.getProperty("shared_wd")
 236         if shared_wd:
 237                 return False
 238         do_cleanup = step.getProperty("do_cleanup")
 239         if do_cleanup:
 240                 return True
 241         else:
 242                 return False
 243
 244 def IsExpireRequested(step):
 245         shared_wd = step.getProperty("shared_wd")
 246         if shared_wd:
 247                 return False
 248         else:
 249                 return not IsCleanupRequested(step)
 250
 251 def IsGitFreshRequested(step):
 252         do_cleanup = step.getProperty("do_cleanup")
 253         if do_cleanup:
 254                 return True
 255         else:
 256                 return False
 257
 258 def IsGitCleanRequested(step):
 259         return not IsGitFreshRequested(step)
 260
 261 def IsTaggingRequested(step):
 262         val = step.getProperty("tag")
 263         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
 264                 return True
 265         else:
 266                 return False
 267
 268 def IsNoTaggingRequested(step):
 269         return not IsTaggingRequested(step)
 270
 271 def IsNoMasterBuild(step):
 272         return repo_branch != "master"
 273
 274 def GetBaseVersion():
 275         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 276                 return repo_branch.split('-')[1]
 277         else:
 278                 return "master"
 279
 280 @properties.renderer
 281 def GetVersionPrefix(props):
 282         basever = GetBaseVersion()
 283         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 284                 return "%s/" % props["tag"]
 285         elif basever != "master":
 286                 return "%s-SNAPSHOT/" % basever
 287         else:
 288                 return ""
 289
 290 @properties.renderer
 291 def GetNumJobs(props):
 292         if props.hasProperty("max_builds") and props.hasProperty("nproc"):
 293                 return str(int(props["nproc"]) / (props["max_builds"] + other_builds))
 294         else:
 295                 return "1"
 296
 297 @properties.renderer
 298 def GetCC(props):
 299         if props.hasProperty("cc_command"):
 300                 return props["cc_command"]
 301         else:
 302                 return "gcc"
 303
 304 @properties.renderer
 305 def GetCXX(props):
 306         if props.hasProperty("cxx_command"):
 307                 return props["cxx_command"]
 308         else:
 309                 return "g++"
 310
 311 @properties.renderer
 312 def GetCwd(props):
 313         if props.hasProperty("builddir"):
 314                 return props["builddir"]
 315         elif props.hasProperty("workdir"):
 316                 return props["workdir"]
 317         else:
 318                 return "/"
 319
 320 @properties.renderer
 321 def GetCCache(props):
 322         if props.hasProperty("ccache_command") and "ccache" in props["ccache_command"]:
 323                 return props["ccache_command"] + " "
 324         else:
 325                 return ""
 326
 327 def GetNextBuild(builder, requests):
 328         for r in requests:
 329                 if r.properties and r.properties.hasProperty("tag"):
 330                         return r
 331         return requests[0]
 332
 333 def MakeEnv(overrides=None, tryccache=False):
 334         if tryccache:
 335                 envcc = Interpolate("%(kw:ccache)s%(kw:cc)s", ccache=GetCCache, cc=GetCC)
 336                 envcxx = Interpolate("%(kw:ccache)s%(kw:cxx)s", ccache=GetCCache, cxx=GetCXX)
 337         else:
 338                 envcc = Interpolate("%(kw:cc)s", cc=GetCC)
 339                 envcxx = Interpolate("%(kw:cxx)s", cxx=GetCXX)
 340         env = {
 341                 'CC': envcc,
 342                 'CXX': envcxx,
 343                 'CCACHE_BASEDIR': Interpolate("%(kw:cwd)s", cwd=GetCwd)
 344         }
 345         if overrides is not None:
 346                 env.update(overrides)
 347         return env
 348
 349 @properties.renderer
 350 def NetLockDl(props):
 351         lock = None
 352         if props.hasProperty("dl_lock"):
 353                 lock = NetLocks[props["dl_lock"]]
 354         if lock is not None:
 355                 return [lock.access('exclusive')]
 356         else:
 357                 return []
 358
 359 @properties.renderer
 360 def NetLockUl(props):
 361         lock = None
 362         if props.hasProperty("ul_lock"):
 363                 lock = NetLocks[props["ul_lock"]]
 364         if lock is not None:
 365                 return [lock.access('exclusive')]
 366         else:
 367                 return []
 368
 369 c['builders'] = []
 370
 371 dlLock = locks.SlaveLock("slave_dl")
 372
 373 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 374         checkBuiltin() {
 375                 local symbol op path file;
 376                 for file in $CHANGED_FILES; do
 377                         case "$file" in
 378                                 package/*/*) : ;;
 379                                 *) return 0 ;;
 380                         esac;
 381                 done;
 382                 while read symbol op path; do
 383                         case "$symbol" in package-*)
 384                                 symbol="${symbol##*(}";
 385                                 symbol="${symbol%)}";
 386                                 for file in $CHANGED_FILES; do
 387                                         case "$file" in "package/$path/"*)
 388                                                 grep -qsx "$symbol=y" .config && return 0
 389                                         ;; esac;
 390                                 done;
 391                         esac;
 392                 done < tmp/.packagedeps;
 393                 return 1;
 394         }
 395 """).strip()
 396
 397
 398 class IfBuiltinShellCommand(ShellCommand):
 399         def _quote(self, str):
 400                 if re.search("[^a-zA-Z0-9/_.-]", str):
 401                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 402                 return str
 403
 404         def setCommand(self, command):
 405                 if not isinstance(command, (str, unicode)):
 406                         command = ' '.join(map(self._quote, command))
 407                 self.command = [
 408                         '/bin/sh', '-c',
 409                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 410                 ]
 411
 412         def setupEnvironment(self, cmd):
 413                 slaveEnv = self.slaveEnvironment
 414                 if slaveEnv is None:
 415                         slaveEnv = { }
 416                 changedFiles = { }
 417                 for request in self.build.requests:
 418                         for source in request.sources:
 419                                 for change in source.changes:
 420                                         for file in change.files:
 421                                                 changedFiles[file] = True
 422                 fullSlaveEnv = slaveEnv.copy()
 423                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 424                 cmd.args['env'] = fullSlaveEnv
 425
 426 slaveNames = [ ]
 427
 428 for slave in c['slaves']:
 429         slaveNames.append(slave.slavename)
 430
 431 for target in targets:
 432         ts = target.split('/')
 433
 434         factory = BuildFactory()
 435
 436         # find number of cores
 437         factory.addStep(SetPropertyFromCommand(
 438                 name = "nproc",
 439                 property = "nproc",
 440                 description = "Finding number of CPUs",
 441                 command = ["nproc"]))
 442
 443         # find gcc and g++ compilers
 444         if cc_version is not None:
 445                 factory.addStep(FileDownload(
 446                         mastersrc = "findbin.pl",
 447                         slavedest = "../findbin.pl",
 448                         mode = 0755))
 449
 450                 factory.addStep(SetPropertyFromCommand(
 451                         name = "gcc",
 452                         property = "cc_command",
 453                         description = "Finding gcc command",
 454                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 455                         haltOnFailure = True))
 456
 457                 factory.addStep(SetPropertyFromCommand(
 458                         name = "g++",
 459                         property = "cxx_command",
 460                         description = "Finding g++ command",
 461                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 462                         haltOnFailure = True))
 463
 464         # see if ccache is available
 465         factory.addStep(SetPropertyFromCommand(
 466                 property = "ccache_command",
 467                 command = ["which", "ccache"],
 468                 description = "Testing for ccache command",
 469                 haltOnFailure = False,
 470                 flunkOnFailure = False,
 471                 warnOnFailure = False,
 472         ))
 473
 474         # expire tree if needed
 475         if tree_expire > 0:
 476                 factory.addStep(FileDownload(
 477                         doStepIf = IsExpireRequested,
 478                         mastersrc = "expire.sh",
 479                         slavedest = "../expire.sh",
 480                         mode = 0755))
 481
 482                 factory.addStep(ShellCommand(
 483                         name = "expire",
 484                         description = "Checking for build tree expiry",
 485                         command = ["./expire.sh", str(tree_expire)],
 486                         workdir = ".",
 487                         haltOnFailure = True,
 488                         doStepIf = IsExpireRequested,
 489                         timeout = 2400))
 490
 491         # cleanup.sh if needed
 492         factory.addStep(FileDownload(
 493                 mastersrc = "cleanup.sh",
 494                 slavedest = "../cleanup.sh",
 495                 mode = 0755,
 496                 doStepIf = IsCleanupRequested))
 497
 498         factory.addStep(ShellCommand(
 499                 name = "cleanold",
 500                 description = "Cleaning previous builds",
 501                 command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:slavename)s"), Interpolate("%(prop:buildername)s"), "full"],
 502                 workdir = ".",
 503                 haltOnFailure = True,
 504                 doStepIf = IsCleanupRequested,
 505                 timeout = 2400))
 506
 507         factory.addStep(ShellCommand(
 508                 name = "cleanup",
 509                 description = "Cleaning work area",
 510                 command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:slavename)s"), Interpolate("%(prop:buildername)s"), "single"],
 511                 workdir = ".",
 512                 haltOnFailure = True,
 513                 doStepIf = IsCleanupRequested,
 514                 timeout = 2400))
 515
 516         # user-requested clean targets
 517         for tuple in CleanTargetMap:
 518                 factory.addStep(ShellCommand(
 519                         name = tuple[1],
 520                         description = 'User-requested "make %s"' % tuple[1],
 521                         command = ["make", tuple[1], "V=s"],
 522                         env = MakeEnv(),
 523                         doStepIf = IsMakeCleanRequested(tuple[0])
 524                 ))
 525
 526         # check out the source
 527         # Git() runs:
 528           # if repo doesn't exist: 'git clone repourl'
 529           # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
 530           # 'git fetch -t repourl branch; git reset --hard revision'
 531         # Git() parameters can't take a renderer until buildbot 0.8.10, so we have to split the fresh and clean cases
 532         # if buildbot is updated, one can use: method = Interpolate('%(prop:do_cleanup:#?|fresh|clean)s')
 533         factory.addStep(Git(
 534                 repourl = repo_url,
 535                 branch = repo_branch,
 536                 mode = 'full',
 537                 method = 'clean',
 538                 locks = NetLockDl,
 539                 haltOnFailure = True,
 540                 doStepIf = IsGitCleanRequested,
 541         ))
 542
 543         factory.addStep(Git(
 544                 repourl = repo_url,
 545                 branch = repo_branch,
 546                 mode = 'full',
 547                 method = 'fresh',
 548                 locks = NetLockDl,
 549                 haltOnFailure = True,
 550                 doStepIf = IsGitFreshRequested,
 551         ))
 552
 553         # update remote refs
 554         factory.addStep(ShellCommand(
 555                 name = "fetchrefs",
 556                 description = "Fetching Git remote refs",
 557                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 558                 haltOnFailure = True
 559         ))
 560
 561         # switch to tag
 562         factory.addStep(ShellCommand(
 563                 name = "switchtag",
 564                 description = "Checking out Git tag",
 565                 command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")],
 566                 haltOnFailure = True,
 567                 doStepIf = IsTaggingRequested
 568         ))
 569
 570         factory.addStep(ShellCommand(
 571                 name = "rmtmp",
 572                 description = "Remove tmp folder",
 573                 command=["rm", "-rf", "tmp/"]))
 574
 575         # feed
 576 #       factory.addStep(ShellCommand(
 577 #               name = "feedsconf",
 578 #               description = "Copy the feeds.conf",
 579 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 580
 581         # feed
 582         factory.addStep(ShellCommand(
 583                 name = "rmfeedlinks",
 584                 description = "Remove feed symlinks",
 585                 command=["rm", "-rf", "package/feeds/"]))
 586
 587         # feed
 588         factory.addStep(ShellCommand(
 589                 name = "updatefeeds",
 590                 description = "Updating feeds",
 591                 command=["./scripts/feeds", "update"],
 592                 env = MakeEnv(),
 593                 locks = NetLockDl,
 594         ))
 595
 596         # feed
 597         factory.addStep(ShellCommand(
 598                 name = "installfeeds",
 599                 description = "Installing feeds",
 600                 command=["./scripts/feeds", "install", "-a"],
 601                 env = MakeEnv()))
 602
 603         # seed config
 604         factory.addStep(FileDownload(
 605                 mastersrc = "config.seed",
 606                 slavedest = ".config",
 607                 mode = 0644
 608         ))
 609
 610         # configure
 611         factory.addStep(ShellCommand(
 612                 name = "newconfig",
 613                 description = "Seeding .config",
 614                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 615         ))
 616
 617         factory.addStep(ShellCommand(
 618                 name = "delbin",
 619                 description = "Removing output directory",
 620                 command = ["rm", "-rf", "bin/"]
 621         ))
 622
 623         factory.addStep(ShellCommand(
 624                 name = "defconfig",
 625                 description = "Populating .config",
 626                 command = ["make", "defconfig"],
 627                 env = MakeEnv()
 628         ))
 629
 630         # check arch
 631         factory.addStep(ShellCommand(
 632                 name = "checkarch",
 633                 description = "Checking architecture",
 634                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 635                 logEnviron = False,
 636                 want_stdout = False,
 637                 want_stderr = False,
 638                 haltOnFailure = True
 639         ))
 640
 641         # find libc suffix
 642         factory.addStep(SetPropertyFromCommand(
 643                 name = "libc",
 644                 property = "libc",
 645                 description = "Finding libc suffix",
 646                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 647
 648         # ccache helper
 649         factory.addStep(FileDownload(
 650                 mastersrc = "ccache.sh",
 651                 slavedest = "../ccache.sh",
 652                 mode = 0755
 653         ))
 654
 655         # ccache prepare
 656         factory.addStep(ShellCommand(
 657                 name = "prepccache",
 658                 description = "Preparing ccache",
 659                 command = ["../ccache.sh"]
 660         ))
 661
 662         # install build key
 663         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 664         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 665
 666         # prepare dl
 667         factory.addStep(ShellCommand(
 668                 name = "dldir",
 669                 description = "Preparing dl/",
 670                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 671                 logEnviron = False,
 672                 want_stdout = False
 673         ))
 674
 675         # prepare tar
 676         factory.addStep(ShellCommand(
 677                 name = "dltar",
 678                 description = "Building and installing GNU tar",
 679                 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
 680                 env = MakeEnv(tryccache=True),
 681                 haltOnFailure = True
 682         ))
 683
 684         # populate dl
 685         factory.addStep(ShellCommand(
 686                 name = "dlrun",
 687                 description = "Populating dl/",
 688                 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "download", "V=s"],
 689                 env = MakeEnv(),
 690                 logEnviron = False,
 691                 locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
 692         ))
 693
 694         factory.addStep(ShellCommand(
 695                 name = "cleanbase",
 696                 description = "Cleaning base-files",
 697                 command=["make", "package/base-files/clean", "V=s"]
 698         ))
 699
 700         # build
 701         factory.addStep(ShellCommand(
 702                 name = "tools",
 703                 description = "Building and installing tools",
 704                 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/install", "V=s"],
 705                 env = MakeEnv(tryccache=True),
 706                 haltOnFailure = True
 707         ))
 708
 709         factory.addStep(ShellCommand(
 710                 name = "toolchain",
 711                 description = "Building and installing toolchain",
 712                 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "toolchain/install", "V=s"],
 713                 env = MakeEnv(),
 714                 haltOnFailure = True
 715         ))
 716
 717         factory.addStep(ShellCommand(
 718                 name = "kmods",
 719                 description = "Building kmods",
 720                 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 721                 env = MakeEnv(),
 722                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 723                 haltOnFailure = True
 724         ))
 725
 726         # find kernel version
 727         factory.addStep(SetPropertyFromCommand(
 728                 name = "kernelversion",
 729                 property = "kernelversion",
 730                 description = "Finding the effective Kernel version",
 731                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
 732                 env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) }
 733         ))
 734
 735         factory.addStep(ShellCommand(
 736                 name = "pkgclean",
 737                 description = "Cleaning up package build",
 738                 command=["make", "package/cleanup", "V=s"]
 739         ))
 740
 741         factory.addStep(ShellCommand(
 742                 name = "pkgbuild",
 743                 description = "Building packages",
 744                 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 745                 env = MakeEnv(),
 746                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 747                 haltOnFailure = True
 748         ))
 749
 750         # factory.addStep(IfBuiltinShellCommand(
 751         factory.addStep(ShellCommand(
 752                 name = "pkginstall",
 753                 description = "Installing packages",
 754                 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/install", "V=s"],
 755                 env = MakeEnv(),
 756                 haltOnFailure = True
 757         ))
 758
 759         factory.addStep(ShellCommand(
 760                 name = "pkgindex",
 761                 description = "Indexing packages",
 762                 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s"],
 763                 env = MakeEnv(),
 764                 haltOnFailure = True
 765         ))
 766
 767         if enable_kmod_archive:
 768                 factory.addStep(ShellCommand(
 769                         name = "kmoddir",
 770                         description = "Creating kmod directory",
 771                         command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
 772                         haltOnFailure = True
 773                 ))
 774
 775                 factory.addStep(ShellCommand(
 776                         name = "kmodprepare",
 777                         description = "Preparing kmod archive",
 778                         command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
 779                                  Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
 780                                  Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 781                         haltOnFailure = True
 782                 ))
 783
 784                 factory.addStep(ShellCommand(
 785                         name = "kmodindex",
 786                         description = "Indexing kmod archive",
 787                         command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s",
 788                                  Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 789                         env = MakeEnv(),
 790                         haltOnFailure = True
 791                 ))
 792
 793                 # find rootfs staging directory
 794                 factory.addStep(SetPropertyFromCommand(
 795                         name = "stageroot",
 796                         property = "stageroot",
 797                         description = "Finding the rootfs staging directory",
 798                         command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"],
 799                         env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) }
 800                 ))
 801
 802                 factory.addStep(ShellCommand(
 803                         name = "filesdir",
 804                         description = "Creating file overlay directory",
 805                         command=["mkdir", "-p", "files/etc/opkg"],
 806                         haltOnFailure = True
 807                 ))
 808
 809                 factory.addStep(ShellCommand(
 810                         name = "kmodconfig",
 811                         description = "Embedding kmod repository configuration",
 812                         command=Interpolate("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(prop:kernelversion)s#' " +
 813                                                "%(prop:stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"),
 814                         haltOnFailure = True
 815                 ))
 816
 817         #factory.addStep(IfBuiltinShellCommand(
 818         factory.addStep(ShellCommand(
 819                 name = "images",
 820                 description = "Building and installing images",
 821                 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/install", "V=s"],
 822                 env = MakeEnv(),
 823                 haltOnFailure = True
 824         ))
 825
 826         factory.addStep(ShellCommand(
 827                 name = "diffconfig",
 828                 description = "Generating config.seed",
 829                 command=["make", "-j1", "diffconfig", "V=s"],
 830                 env = MakeEnv(),
 831                 haltOnFailure = True
 832         ))
 833
 834         factory.addStep(ShellCommand(
 835                 name = "checksums",
 836                 description = "Calculating checksums",
 837                 command=["make", "-j1", "checksum", "V=s"],
 838                 env = MakeEnv(),
 839                 haltOnFailure = True
 840         ))
 841
 842         # sign
 843         if gpg_keyid is not None:
 844                 factory.addStep(MasterShellCommand(
 845                         name = "signprepare",
 846                         description = "Preparing temporary signing directory",
 847                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 848                         haltOnFailure = True
 849                 ))
 850
 851                 factory.addStep(ShellCommand(
 852                         name = "signpack",
 853                         description = "Packing files to sign",
 854                         command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
 855                         haltOnFailure = True
 856                 ))
 857
 858                 factory.addStep(FileUpload(
 859                         slavesrc = "sign.tar.gz",
 860                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 861                         haltOnFailure = True
 862                 ))
 863
 864                 factory.addStep(MasterShellCommand(
 865                         name = "signfiles",
 866                         description = "Signing files",
 867                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 868                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 869                         haltOnFailure = True
 870                 ))
 871
 872                 factory.addStep(FileDownload(
 873                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 874                         slavedest = "sign.tar.gz",
 875                         haltOnFailure = True
 876                 ))
 877
 878                 factory.addStep(ShellCommand(
 879                         name = "signunpack",
 880                         description = "Unpacking signed files",
 881                         command = ["tar", "-xzf", "sign.tar.gz"],
 882                         haltOnFailure = True
 883                 ))
 884
 885         # upload
 886         factory.addStep(ShellCommand(
 887                 name = "dirprepare",
 888                 description = "Preparing upload directory structure",
 889                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 890                 haltOnFailure = True
 891         ))
 892
 893         factory.addStep(ShellCommand(
 894                 name = "linkprepare",
 895                 description = "Preparing repository symlink",
 896                 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=GetBaseVersion()), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
 897                 doStepIf = IsNoMasterBuild,
 898                 haltOnFailure = True
 899         ))
 900
 901         if enable_kmod_archive:
 902                 factory.addStep(ShellCommand(
 903                         name = "kmoddirprepare",
 904                         description = "Preparing kmod archive upload directory",
 905                         command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 906                         haltOnFailure = True
 907                 ))
 908
 909         factory.addStep(ShellCommand(
 910                 name = "dirupload",
 911                 description = "Uploading directory structure",
 912                 command = ["rsync", "-4", "--info=name", "-az", "tmp/upload/", "%s/" %(rsync_bin_url)],
 913                 env={'RSYNC_PASSWORD': rsync_bin_key},
 914                 haltOnFailure = True,
 915                 logEnviron = False,
 916                 locks = NetLockUl,
 917         ))
 918
 919         factory.addStep(FileDownload(
 920                         mastersrc = "rsync.sh",
 921                         slavedest = "../rsync.sh",
 922                         mode = 0755))
 923
 924         factory.addStep(ShellCommand(
 925                 name = "targetupload",
 926                 description = "Uploading target files",
 927                 command=["../rsync.sh", "-4", "--info=name", "--exclude=/kmods/", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 928                          "-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
 929                          Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 930                 env={'RSYNC_PASSWORD': rsync_bin_key},
 931                 haltOnFailure = True,
 932                 logEnviron = False,
 933                 locks = NetLockUl,
 934         ))
 935
 936         if enable_kmod_archive:
 937                 factory.addStep(ShellCommand(
 938                         name = "kmodupload",
 939                         description = "Uploading kmod archive",
 940                         command=["../rsync.sh", "-4", "--info=name", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 941                                  "-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
 942                                  Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 943                         env={'RSYNC_PASSWORD': rsync_bin_key},
 944                         haltOnFailure = True,
 945                         logEnviron = False,
 946                         locks = NetLockUl,
 947                 ))
 948
 949         if rsync_src_url is not None:
 950                 factory.addStep(ShellCommand(
 951                         name = "sourceupload",
 952                         description = "Uploading source archives",
 953                         command=["../rsync.sh", "-4", "--info=name", "--size-only", "--delay-updates",
 954                                  Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:slavename)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
 955                         env={'RSYNC_PASSWORD': rsync_src_key},
 956                         haltOnFailure = True,
 957                         logEnviron = False,
 958                         locks = NetLockUl,
 959                 ))
 960
 961         if False:
 962                 factory.addStep(ShellCommand(
 963                         name = "packageupload",
 964                         description = "Uploading package files",
 965                         command=["../rsync.sh", "-4", "--info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 966                         env={'RSYNC_PASSWORD': rsync_bin_key},
 967                         haltOnFailure = False,
 968                         logEnviron = False,
 969                         locks = NetLockUl,
 970                 ))
 971
 972         # logs
 973         if False:
 974                 factory.addStep(ShellCommand(
 975                         name = "upload",
 976                         description = "Uploading logs",
 977                         command=["../rsync.sh", "-4", "-info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 978                         env={'RSYNC_PASSWORD': rsync_bin_key},
 979                         haltOnFailure = False,
 980                         alwaysRun = True,
 981                         logEnviron = False,
 982                         locks = NetLockUl,
 983                 ))
 984
 985         factory.addStep(ShellCommand(
 986                 name = "df",
 987                 description = "Reporting disk usage",
 988                 command=["df", "-h", "."],
 989                 env={'LC_ALL': 'C'},
 990                 haltOnFailure = False,
 991                 alwaysRun = True
 992         ))
 993
 994         factory.addStep(ShellCommand(
 995                 name = "ccachestat",
 996                 description = "Reporting ccache stats",
 997                 command=["ccache", "-s"],
 998                 env = MakeEnv(overrides={ 'PATH': ["./staging_dir/host/bin", "${PATH}"] }),
 999                 want_stderr = False,
1000                 haltOnFailure = False,
1001                 flunkOnFailure = False,
1002                 warnOnFailure = False,
1003                 alwaysRun = True,
1004         ))
1005
1006         from buildbot.config import BuilderConfig
1007
1008         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))
1009
1010
1011 ####### STATUS TARGETS
1012
1013 # 'status' is a list of Status Targets. The results of each build will be
1014 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1015 # including web pages, email senders, and IRC bots.
1016
1017 c['status'] = []
1018
1019 from buildbot.status import html
1020 from buildbot.status.web import authz, auth
1021
1022 if ini.has_option("status", "bind"):
1023         if ini.has_option("status", "user") and ini.has_option("status", "password"):
1024                 authz_cfg=authz.Authz(
1025                         # change any of these to True to enable; see the manual for more
1026                         # options
1027                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
1028                         gracefulShutdown = 'auth',
1029                         forceBuild = 'auth', # use this to test your slave once it is set up
1030                         forceAllBuilds = 'auth',
1031                         pingBuilder = False,
1032                         stopBuild = 'auth',
1033                         stopAllBuilds = 'auth',
1034                         cancelPendingBuild = 'auth',
1035                 )
1036                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
1037         else:
1038                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
1039
1040
1041 from buildbot.status import words
1042
1043 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
1044         irc_host = ini.get("irc", "host")
1045         irc_port = 6667
1046         irc_chan = ini.get("irc", "channel")
1047         irc_nick = ini.get("irc", "nickname")
1048         irc_pass = None
1049
1050         if ini.has_option("irc", "port"):
1051                 irc_port = ini.getint("irc", "port")
1052
1053         if ini.has_option("irc", "password"):
1054                 irc_pass = ini.get("irc", "password")
1055
1056         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
1057                         channels = [{ "channel": irc_chan }],
1058                         notify_events = {
1059                           'exception': 1,
1060                           'successToFailure': 1,
1061                           'failureToSuccess': 1
1062                         }
1063         )
1064
1065         c['status'].append(irc)
1066
1067 ####### DB URL
1068
1069 c['db'] = {
1070         # This specifies what database buildbot uses to store its state.  You can leave
1071         # this at its default for all but the largest installations.
1072         'db_url' : "sqlite:///state.sqlite",
1073 }