phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### PROJECT IDENTITY
  22
  23 # the 'title' string will appear at the top of this buildbot
  24 # installation's html.WebStatus home page (linked to the
  25 # 'titleURL') and is embedded in the title of the waterfall HTML page.
  26
  27 c['title'] = ini.get("general", "title")
  28 c['titleURL'] = ini.get("general", "title_url")
  29
  30 # the 'buildbotURL' string should point to the location where the buildbot's
  31 # internal web server (usually the html.WebStatus page) is visible. This
  32 # typically uses the port number set in the Waterfall 'status' entry, but
  33 # with an externally-visible host name which the buildbot cannot figure out
  34 # without some help.
  35
  36 c['buildbotURL'] = ini.get("general", "buildbot_url")
  37
  38 ####### BUILDSLAVES
  39
  40 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  41 # a BuildSlave object, specifying a unique slave name and password.  The same
  42 # slave name and password must be configured on the slave.
  43 from buildbot.buildslave import BuildSlave
  44
  45 slave_port = 9989
  46
  47 if ini.has_option("general", "port"):
  48         slave_port = ini.getint("general", "port")
  49
  50 c['slaves'] = []
  51 max_builds = dict()
  52 NetLocks = dict()
  53
  54 for section in ini.sections():
  55         if section.startswith("slave "):
  56                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  57                         sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False }
  58                         name = ini.get(section, "name")
  59                         password = ini.get(section, "password")
  60                         max_builds[name] = 1
  61                         if ini.has_option(section, "builds"):
  62                                 max_builds[name] = ini.getint(section, "builds")
  63                         if ini.has_option(section, "cleanup"):
  64                                 sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
  65                         if ini.has_option(section, "dl_lock"):
  66                                 lockname = ini.get(section, "dl_lock")
  67                                 sl_props['dl_lock'] = lockname
  68                                 if lockname not in NetLocks:
  69                                         NetLocks[lockname] = locks.MasterLock(lockname)
  70                         if ini.has_option(section, "ul_lock"):
  71                                 lockname = ini.get(section, "dl_lock")
  72                                 sl_props['ul_lock'] = lockname
  73                                 if lockname not in NetLocks:
  74                                         NetLocks[lockname] = locks.MasterLock(lockname)
  75                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name], properties = sl_props))
  76
  77 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  78 # This must match the value configured into the buildslaves (with their
  79 # --master option)
  80 c['slavePortnum'] = slave_port
  81
  82 # coalesce builds
  83 c['mergeRequests'] = True
  84
  85 # Reduce amount of backlog data
  86 c['buildHorizon'] = 30
  87 c['logHorizon'] = 20
  88
  89 ####### CHANGESOURCES
  90
  91 home_dir = os.path.abspath(ini.get("general", "homedir"))
  92 tree_expire = 0
  93 other_builds = 0
  94 cc_version = None
  95
  96 cc_command = "gcc"
  97 cxx_command = "g++"
  98
  99 if ini.has_option("general", "expire"):
 100         tree_expire = ini.getint("general", "expire")
 101
 102 if ini.has_option("general", "other_builds"):
 103         other_builds = ini.getint("general", "other_builds")
 104
 105 if ini.has_option("general", "cc_version"):
 106         cc_version = ini.get("general", "cc_version").split()
 107         if len(cc_version) == 1:
 108                 cc_version = ["eq", cc_version[0]]
 109
 110 repo_url = ini.get("repo", "url")
 111 repo_branch = "master"
 112
 113 if ini.has_option("repo", "branch"):
 114         repo_branch = ini.get("repo", "branch")
 115
 116 rsync_bin_url = ini.get("rsync", "binary_url")
 117 rsync_bin_key = ini.get("rsync", "binary_password")
 118
 119 rsync_src_url = None
 120 rsync_src_key = None
 121
 122 if ini.has_option("rsync", "source_url"):
 123         rsync_src_url = ini.get("rsync", "source_url")
 124         rsync_src_key = ini.get("rsync", "source_password")
 125
 126 gpg_home = "~/.gnupg"
 127 gpg_keyid = None
 128 gpg_comment = "Unattended build signature"
 129 gpg_passfile = "/dev/null"
 130
 131 if ini.has_option("gpg", "home"):
 132         gpg_home = ini.get("gpg", "home")
 133
 134 if ini.has_option("gpg", "keyid"):
 135         gpg_keyid = ini.get("gpg", "keyid")
 136
 137 if ini.has_option("gpg", "comment"):
 138         gpg_comment = ini.get("gpg", "comment")
 139
 140 if ini.has_option("gpg", "passfile"):
 141         gpg_passfile = ini.get("gpg", "passfile")
 142
 143 enable_kmod_archive = True
 144
 145
 146 # find targets
 147 targets = [ ]
 148
 149 if not os.path.isdir(home_dir+'/source.git'):
 150         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 151 else:
 152         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 153
 154 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 155         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 156
 157 while True:
 158         line = findtargets.stdout.readline()
 159         if not line:
 160                 break
 161         ta = line.strip().split(' ')
 162         targets.append(ta[0])
 163
 164
 165 # the 'change_source' setting tells the buildmaster how it should find out
 166 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 167
 168 from buildbot.changes.gitpoller import GitPoller
 169 c['change_source'] = []
 170 c['change_source'].append(GitPoller(
 171         repo_url,
 172         workdir=home_dir+'/work.git', branch=repo_branch,
 173         pollinterval=300))
 174
 175 ####### SCHEDULERS
 176
 177 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 178 # case, just kick off a 'basebuild' build
 179
 180 from buildbot.schedulers.basic import SingleBranchScheduler
 181 from buildbot.schedulers.forcesched import ForceScheduler
 182 from buildbot.changes import filter
 183 c['schedulers'] = []
 184 c['schedulers'].append(SingleBranchScheduler(
 185         name="all",
 186         change_filter=filter.ChangeFilter(branch=repo_branch),
 187         treeStableTimer=60,
 188         builderNames=targets))
 189
 190 c['schedulers'].append(ForceScheduler(
 191         name="force",
 192         builderNames=targets))
 193
 194 ####### BUILDERS
 195
 196 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 197 # what steps, and which slaves can execute them.  Note that any particular build will
 198 # only take place on one slave.
 199
 200 from buildbot.process.factory import BuildFactory
 201 from buildbot.steps.source.git import Git
 202 from buildbot.steps.shell import ShellCommand
 203 from buildbot.steps.shell import SetProperty
 204 from buildbot.steps.shell import SetPropertyFromCommand
 205 from buildbot.steps.transfer import FileUpload
 206 from buildbot.steps.transfer import FileDownload
 207 from buildbot.steps.master import MasterShellCommand
 208 from buildbot.process.properties import WithProperties
 209 from buildbot.process import properties
 210
 211
 212 CleanTargetMap = [
 213         [ "tools",      "tools/clean"                   ],
 214         [ "chain",      "toolchain/clean"               ],
 215         [ "linux",      "target/linux/clean"    ],
 216         [ "dir",        "dirclean"                              ],
 217         [ "dist",       "distclean"                             ]
 218 ]
 219
 220 def IsMakeCleanRequested(pattern):
 221         def CheckCleanProperty(step):
 222                 val = step.getProperty("clean")
 223                 if val and re.match(pattern, val):
 224                         return True
 225                 else:
 226                         return False
 227
 228         return CheckCleanProperty
 229
 230 def IsCleanupRequested(step):
 231         do_cleanup = step.getProperty("do_cleanup")
 232         if do_cleanup:
 233                 return True
 234         else:
 235                 return False
 236
 237 def IsExpireRequested(step):
 238         return not IsCleanupRequested(step)
 239
 240 def IsTaggingRequested(step):
 241         val = step.getProperty("tag")
 242         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
 243                 return True
 244         else:
 245                 return False
 246
 247 def IsNoTaggingRequested(step):
 248         return not IsTaggingRequested(step)
 249
 250 def IsNoMasterBuild(step):
 251         return repo_branch != "master"
 252
 253 def IsCleanupConfigured(step):
 254         do_cleanup = step.getProperty("do_cleanup")
 255         if do_cleanup:
 256                 return do_cleanup > 0
 257         else:
 258                 return False
 259
 260 def GetBaseVersion(props):
 261         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 262                 return repo_branch.split('-')[1]
 263         else:
 264                 return "master"
 265
 266 def GetVersionPrefix(props):
 267         basever = GetBaseVersion(props)
 268         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 269                 return "%s/" % props["tag"]
 270         elif basever != "master":
 271                 return "%s-SNAPSHOT/" % basever
 272         else:
 273                 return ""
 274
 275 def GetNumJobs(props):
 276         if props.hasProperty("slavename") and props.hasProperty("nproc"):
 277                 return (int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds))
 278         else:
 279                 return 1
 280
 281 def GetCC(props):
 282         if props.hasProperty("cc_command"):
 283                 return props["cc_command"]
 284         else:
 285                 return "gcc"
 286
 287 def GetCXX(props):
 288         if props.hasProperty("cxx_command"):
 289                 return props["cxx_command"]
 290         else:
 291                 return "g++"
 292
 293 def GetCwd(props):
 294         if props.hasProperty("builddir"):
 295                 return props["builddir"]
 296         elif props.hasProperty("workdir"):
 297                 return props["workdir"]
 298         else:
 299                 return "/"
 300
 301 def GetCCache(props):
 302         if props.hasProperty("ccache_command") and "ccache" in props["ccache_command"]:
 303                 return props["ccache_command"] + " "
 304         else:
 305                 return ""
 306
 307 def GetNextBuild(builder, requests):
 308         for r in requests:
 309                 if r.properties and r.properties.hasProperty("tag"):
 310                         return r
 311         return requests[0]
 312
 313 def MakeEnv(overrides=None, tryccache=False):
 314         if tryccache:
 315                 envcc = Interpolate("%(kw:ccache)s%(kw:cc)s", ccache=GetCCache, cc=GetCC)
 316                 envcxx = Interpolate("%(kw:ccache)s%(kw:cxx)s", ccache=GetCCache, cxx=GetCXX)
 317         else:
 318                 envcc = Interpolate("%(kw:cc)s", cc=GetCC)
 319                 envcxx = Interpolate("%(kw:cxx)s", cxx=GetCXX)
 320         env = {
 321                 'CC': envcc,
 322                 'CXX': envcxx,
 323                 'CCACHE_BASEDIR': Interpolate("%(kw:cwd)s", cwd=GetCwd)
 324         }
 325         if overrides is not None:
 326                 env.update(overrides)
 327         return env
 328
 329 @properties.renderer
 330 def NetLockDl(props):
 331         lock = None
 332         if props.hasProperty("dl_lock"):
 333                 lock = NetLocks[props["dl_lock"]]
 334         if lock is not None:
 335                 return [lock.access('exclusive')]
 336         else:
 337                 return []
 338
 339 @properties.renderer
 340 def NetLockUl(props):
 341         lock = None
 342         if props.hasProperty("ul_lock"):
 343                 lock = NetLocks[props["ul_lock"]]
 344         if lock is not None:
 345                 return [lock.access('exclusive')]
 346         else:
 347                 return []
 348
 349 c['builders'] = []
 350
 351 dlLock = locks.SlaveLock("slave_dl")
 352
 353 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 354         checkBuiltin() {
 355                 local symbol op path file;
 356                 for file in $CHANGED_FILES; do
 357                         case "$file" in
 358                                 package/*/*) : ;;
 359                                 *) return 0 ;;
 360                         esac;
 361                 done;
 362                 while read symbol op path; do
 363                         case "$symbol" in package-*)
 364                                 symbol="${symbol##*(}";
 365                                 symbol="${symbol%)}";
 366                                 for file in $CHANGED_FILES; do
 367                                         case "$file" in "package/$path/"*)
 368                                                 grep -qsx "$symbol=y" .config && return 0
 369                                         ;; esac;
 370                                 done;
 371                         esac;
 372                 done < tmp/.packagedeps;
 373                 return 1;
 374         }
 375 """).strip()
 376
 377
 378 class IfBuiltinShellCommand(ShellCommand):
 379         def _quote(self, str):
 380                 if re.search("[^a-zA-Z0-9/_.-]", str):
 381                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 382                 return str
 383
 384         def setCommand(self, command):
 385                 if not isinstance(command, (str, unicode)):
 386                         command = ' '.join(map(self._quote, command))
 387                 self.command = [
 388                         '/bin/sh', '-c',
 389                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 390                 ]
 391
 392         def setupEnvironment(self, cmd):
 393                 slaveEnv = self.slaveEnvironment
 394                 if slaveEnv is None:
 395                         slaveEnv = { }
 396                 changedFiles = { }
 397                 for request in self.build.requests:
 398                         for source in request.sources:
 399                                 for change in source.changes:
 400                                         for file in change.files:
 401                                                 changedFiles[file] = True
 402                 fullSlaveEnv = slaveEnv.copy()
 403                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 404                 cmd.args['env'] = fullSlaveEnv
 405
 406 slaveNames = [ ]
 407
 408 for slave in c['slaves']:
 409         slaveNames.append(slave.slavename)
 410
 411 for target in targets:
 412         ts = target.split('/')
 413
 414         factory = BuildFactory()
 415
 416         # find number of cores
 417         factory.addStep(SetProperty(
 418                 name = "nproc",
 419                 property = "nproc",
 420                 description = "Finding number of CPUs",
 421                 command = ["nproc"]))
 422
 423         # find gcc and g++ compilers
 424         if cc_version is not None:
 425                 factory.addStep(FileDownload(
 426                         mastersrc = "findbin.pl",
 427                         slavedest = "../findbin.pl",
 428                         mode = 0755))
 429
 430                 factory.addStep(SetProperty(
 431                         name = "gcc",
 432                         property = "cc_command",
 433                         description = "Finding gcc command",
 434                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 435                         haltOnFailure = True))
 436
 437                 factory.addStep(SetProperty(
 438                         name = "g++",
 439                         property = "cxx_command",
 440                         description = "Finding g++ command",
 441                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 442                         haltOnFailure = True))
 443
 444         # see if ccache is available
 445         factory.addStep(SetPropertyFromCommand(
 446                 property = "ccache_command",
 447                 command = ["which", "ccache"],
 448                 description = "Testing for ccache command",
 449                 haltOnFailure = False,
 450                 flunkOnFailure = False,
 451                 warnOnFailure = False,
 452         ))
 453
 454         # expire tree if needed
 455         if tree_expire > 0:
 456                 factory.addStep(FileDownload(
 457                         doStepIf = IsExpireRequested,
 458                         mastersrc = "expire.sh",
 459                         slavedest = "../expire.sh",
 460                         mode = 0755))
 461
 462                 factory.addStep(ShellCommand(
 463                         name = "expire",
 464                         description = "Checking for build tree expiry",
 465                         command = ["./expire.sh", str(tree_expire)],
 466                         workdir = ".",
 467                         haltOnFailure = True,
 468                         doStepIf = IsExpireRequested,
 469                         timeout = 2400))
 470
 471         # cleanup.sh if needed
 472         factory.addStep(FileDownload(
 473                 mastersrc = "cleanup.sh",
 474                 slavedest = "../cleanup.sh",
 475                 mode = 0755,
 476                 doStepIf = IsCleanupRequested))
 477
 478         factory.addStep(ShellCommand(
 479                 name = "cleanold",
 480                 description = "Cleaning previous builds",
 481                 command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 482                 workdir = ".",
 483                 haltOnFailure = True,
 484                 doStepIf = IsCleanupRequested,
 485                 timeout = 2400))
 486
 487         factory.addStep(ShellCommand(
 488                 name = "cleanup",
 489                 description = "Cleaning work area",
 490                 command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 491                 workdir = ".",
 492                 haltOnFailure = True,
 493                 doStepIf = IsCleanupRequested,
 494                 timeout = 2400))
 495
 496         # user-requested clean targets
 497         for tuple in CleanTargetMap:
 498                 factory.addStep(ShellCommand(
 499                         name = tuple[1],
 500                         description = 'User-requested "make %s"' % tuple[1],
 501                         command = ["make", tuple[1], "V=s"],
 502                         env = MakeEnv(),
 503                         doStepIf = IsMakeCleanRequested(tuple[0])
 504                 ))
 505
 506         # check out the source
 507         # Git() runs:
 508           # if repo doesn't exist: 'git clone repourl'
 509           # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
 510           # 'git fetch -t repourl branch; git reset --hard revision'
 511         factory.addStep(Git(
 512                 repourl = repo_url,
 513                 branch = repo_branch,
 514                 mode = 'full',
 515                 method = 'clean',
 516                 locks = NetLockDl,
 517                 haltOnFailure = True,
 518         ))
 519
 520         # update remote refs
 521         factory.addStep(ShellCommand(
 522                 name = "fetchrefs",
 523                 description = "Fetching Git remote refs",
 524                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 525                 haltOnFailure = True
 526         ))
 527
 528         # switch to tag
 529         factory.addStep(ShellCommand(
 530                 name = "switchtag",
 531                 description = "Checking out Git tag",
 532                 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
 533                 haltOnFailure = True,
 534                 doStepIf = IsTaggingRequested
 535         ))
 536
 537         factory.addStep(ShellCommand(
 538                 name = "rmtmp",
 539                 description = "Remove tmp folder",
 540                 command=["rm", "-rf", "tmp/"]))
 541
 542         # feed
 543 #       factory.addStep(ShellCommand(
 544 #               name = "feedsconf",
 545 #               description = "Copy the feeds.conf",
 546 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 547
 548         # feed
 549         factory.addStep(ShellCommand(
 550                 name = "rmfeedlinks",
 551                 description = "Remove feed symlinks",
 552                 command=["rm", "-rf", "package/feeds/"]))
 553
 554         # feed
 555         factory.addStep(ShellCommand(
 556                 name = "updatefeeds",
 557                 description = "Updating feeds",
 558                 command=["./scripts/feeds", "update"],
 559                 env = MakeEnv(),
 560                 locks = NetLockDl,
 561         ))
 562
 563         # feed
 564         factory.addStep(ShellCommand(
 565                 name = "installfeeds",
 566                 description = "Installing feeds",
 567                 command=["./scripts/feeds", "install", "-a"],
 568                 env = MakeEnv()))
 569
 570         # seed config
 571         factory.addStep(FileDownload(
 572                 mastersrc = "config.seed",
 573                 slavedest = ".config",
 574                 mode = 0644
 575         ))
 576
 577         # configure
 578         factory.addStep(ShellCommand(
 579                 name = "newconfig",
 580                 description = "Seeding .config",
 581                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 582         ))
 583
 584         factory.addStep(ShellCommand(
 585                 name = "delbin",
 586                 description = "Removing output directory",
 587                 command = ["rm", "-rf", "bin/"]
 588         ))
 589
 590         factory.addStep(ShellCommand(
 591                 name = "defconfig",
 592                 description = "Populating .config",
 593                 command = ["make", "defconfig"],
 594                 env = MakeEnv()
 595         ))
 596
 597         # check arch
 598         factory.addStep(ShellCommand(
 599                 name = "checkarch",
 600                 description = "Checking architecture",
 601                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 602                 logEnviron = False,
 603                 want_stdout = False,
 604                 want_stderr = False,
 605                 haltOnFailure = True
 606         ))
 607
 608         # find libc suffix
 609         factory.addStep(SetProperty(
 610                 name = "libc",
 611                 property = "libc",
 612                 description = "Finding libc suffix",
 613                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 614
 615         # ccache helper
 616         factory.addStep(FileDownload(
 617                 mastersrc = "ccache.sh",
 618                 slavedest = "../ccache.sh",
 619                 mode = 0755
 620         ))
 621
 622         # ccache prepare
 623         factory.addStep(ShellCommand(
 624                 name = "prepccache",
 625                 description = "Preparing ccache",
 626                 command = ["../ccache.sh"]
 627         ))
 628
 629         # install build key
 630         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 631         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 632
 633         # prepare dl
 634         factory.addStep(ShellCommand(
 635                 name = "dldir",
 636                 description = "Preparing dl/",
 637                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 638                 logEnviron = False,
 639                 want_stdout = False
 640         ))
 641
 642         # prepare tar
 643         factory.addStep(ShellCommand(
 644                 name = "dltar",
 645                 description = "Building and installing GNU tar",
 646                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
 647                 env = MakeEnv(tryccache=True),
 648                 haltOnFailure = True
 649         ))
 650
 651         # populate dl
 652         factory.addStep(ShellCommand(
 653                 name = "dlrun",
 654                 description = "Populating dl/",
 655                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
 656                 env = MakeEnv(),
 657                 logEnviron = False,
 658                 locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
 659         ))
 660
 661         factory.addStep(ShellCommand(
 662                 name = "cleanbase",
 663                 description = "Cleaning base-files",
 664                 command=["make", "package/base-files/clean", "V=s"]
 665         ))
 666
 667         # build
 668         factory.addStep(ShellCommand(
 669                 name = "tools",
 670                 description = "Building and installing tools",
 671                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
 672                 env = MakeEnv(tryccache=True),
 673                 haltOnFailure = True
 674         ))
 675
 676         factory.addStep(ShellCommand(
 677                 name = "toolchain",
 678                 description = "Building and installing toolchain",
 679                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
 680                 env = MakeEnv(),
 681                 haltOnFailure = True
 682         ))
 683
 684         factory.addStep(ShellCommand(
 685                 name = "kmods",
 686                 description = "Building kmods",
 687                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 688                 env = MakeEnv(),
 689                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 690                 haltOnFailure = True
 691         ))
 692
 693         # find kernel version
 694         factory.addStep(SetProperty(
 695                 name = "kernelversion",
 696                 property = "kernelversion",
 697                 description = "Finding the effective Kernel version",
 698                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
 699                 env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) }
 700         ))
 701
 702         factory.addStep(ShellCommand(
 703                 name = "pkgclean",
 704                 description = "Cleaning up package build",
 705                 command=["make", "package/cleanup", "V=s"]
 706         ))
 707
 708         factory.addStep(ShellCommand(
 709                 name = "pkgbuild",
 710                 description = "Building packages",
 711                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 712                 env = MakeEnv(),
 713                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 714                 haltOnFailure = True
 715         ))
 716
 717         # factory.addStep(IfBuiltinShellCommand(
 718         factory.addStep(ShellCommand(
 719                 name = "pkginstall",
 720                 description = "Installing packages",
 721                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
 722                 env = MakeEnv(),
 723                 haltOnFailure = True
 724         ))
 725
 726         factory.addStep(ShellCommand(
 727                 name = "pkgindex",
 728                 description = "Indexing packages",
 729                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
 730                 env = MakeEnv(),
 731                 haltOnFailure = True
 732         ))
 733
 734         if enable_kmod_archive:
 735                 factory.addStep(ShellCommand(
 736                         name = "kmoddir",
 737                         description = "Creating kmod directory",
 738                         command=["mkdir", "-p", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s" %(ts[0], ts[1]))],
 739                         haltOnFailure = True
 740                 ))
 741
 742                 factory.addStep(ShellCommand(
 743                         name = "kmodprepare",
 744                         description = "Preparing kmod archive",
 745                         command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
 746                                  WithProperties("bin/targets/%s/%s%%(libc)s/packages/" %(ts[0], ts[1])),
 747                                  WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))],
 748                         haltOnFailure = True
 749                 ))
 750
 751                 factory.addStep(ShellCommand(
 752                         name = "kmodindex",
 753                         description = "Indexing kmod archive",
 754                         command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s",
 755                                  WithProperties("PACKAGE_SUBDIRS=bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))],
 756                         env = MakeEnv(),
 757                         haltOnFailure = True
 758                 ))
 759
 760                 # find rootfs staging directory
 761                 factory.addStep(SetProperty(
 762                         name = "stageroot",
 763                         property = "stageroot",
 764                         description = "Finding the rootfs staging directory",
 765                         command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"],
 766                         env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) }
 767                 ))
 768
 769                 factory.addStep(ShellCommand(
 770                         name = "filesdir",
 771                         description = "Creating file overlay directory",
 772                         command=["mkdir", "-p", "files/etc/opkg"],
 773                         haltOnFailure = True
 774                 ))
 775
 776                 factory.addStep(ShellCommand(
 777                         name = "kmodconfig",
 778                         description = "Embedding kmod repository configuration",
 779                         command=WithProperties("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(kernelversion)s#' " +
 780                                                "%(stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"),
 781                         haltOnFailure = True
 782                 ))
 783
 784         #factory.addStep(IfBuiltinShellCommand(
 785         factory.addStep(ShellCommand(
 786                 name = "images",
 787                 description = "Building and installing images",
 788                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
 789                 env = MakeEnv(),
 790                 haltOnFailure = True
 791         ))
 792
 793         factory.addStep(ShellCommand(
 794                 name = "diffconfig",
 795                 description = "Generating config.seed",
 796                 command=["make", "-j1", "diffconfig", "V=s"],
 797                 env = MakeEnv(),
 798                 haltOnFailure = True
 799         ))
 800
 801         factory.addStep(ShellCommand(
 802                 name = "checksums",
 803                 description = "Calculating checksums",
 804                 command=["make", "-j1", "checksum", "V=s"],
 805                 env = MakeEnv(),
 806                 haltOnFailure = True
 807         ))
 808
 809         # sign
 810         if gpg_keyid is not None:
 811                 factory.addStep(MasterShellCommand(
 812                         name = "signprepare",
 813                         description = "Preparing temporary signing directory",
 814                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 815                         haltOnFailure = True
 816                 ))
 817
 818                 factory.addStep(ShellCommand(
 819                         name = "signpack",
 820                         description = "Packing files to sign",
 821                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ bin/targets/%s/%s%%(libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1], ts[0], ts[1])),
 822                         haltOnFailure = True
 823                 ))
 824
 825                 factory.addStep(FileUpload(
 826                         slavesrc = "sign.tar.gz",
 827                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 828                         haltOnFailure = True
 829                 ))
 830
 831                 factory.addStep(MasterShellCommand(
 832                         name = "signfiles",
 833                         description = "Signing files",
 834                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 835                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 836                         haltOnFailure = True
 837                 ))
 838
 839                 factory.addStep(FileDownload(
 840                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 841                         slavedest = "sign.tar.gz",
 842                         haltOnFailure = True
 843                 ))
 844
 845                 factory.addStep(ShellCommand(
 846                         name = "signunpack",
 847                         description = "Unpacking signed files",
 848                         command = ["tar", "-xzf", "sign.tar.gz"],
 849                         haltOnFailure = True
 850                 ))
 851
 852         # upload
 853         factory.addStep(ShellCommand(
 854                 name = "dirprepare",
 855                 description = "Preparing upload directory structure",
 856                 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 857                 haltOnFailure = True
 858         ))
 859
 860         factory.addStep(ShellCommand(
 861                 name = "linkprepare",
 862                 description = "Preparing repository symlink",
 863                 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
 864                 doStepIf = IsNoMasterBuild,
 865                 haltOnFailure = True
 866         ))
 867
 868         if enable_kmod_archive:
 869                 factory.addStep(ShellCommand(
 870                         name = "kmoddirprepare",
 871                         description = "Preparing kmod archive upload directory",
 872                         command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 873                         haltOnFailure = True
 874                 ))
 875
 876         factory.addStep(ShellCommand(
 877                 name = "dirupload",
 878                 description = "Uploading directory structure",
 879                 command = ["rsync", "-4", "--info=name", "-az", "tmp/upload/", "%s/" %(rsync_bin_url)],
 880                 env={'RSYNC_PASSWORD': rsync_bin_key},
 881                 haltOnFailure = True,
 882                 logEnviron = False,
 883                 locks = NetLockUl,
 884         ))
 885
 886         factory.addStep(FileDownload(
 887                         mastersrc = "rsync.sh",
 888                         slavedest = "../rsync.sh",
 889                         mode = 0755))
 890
 891         factory.addStep(ShellCommand(
 892                 name = "targetupload",
 893                 description = "Uploading target files",
 894                 command=["../rsync.sh", "-4", "--info=name", "--exclude=/kmods/", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 895                          "-a", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
 896                          WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 897                 env={'RSYNC_PASSWORD': rsync_bin_key},
 898                 haltOnFailure = True,
 899                 logEnviron = False,
 900                 locks = NetLockUl,
 901         ))
 902
 903         if enable_kmod_archive:
 904                 factory.addStep(ShellCommand(
 905                         name = "kmodupload",
 906                         description = "Uploading kmod archive",
 907                         command=["../rsync.sh", "-4", "--info=name", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 908                                  "-a", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])),
 909                                  WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 910                         env={'RSYNC_PASSWORD': rsync_bin_key},
 911                         haltOnFailure = True,
 912                         logEnviron = False,
 913                         locks = NetLockUl,
 914                 ))
 915
 916         if rsync_src_url is not None:
 917                 factory.addStep(ShellCommand(
 918                         name = "sourceupload",
 919                         description = "Uploading source archives",
 920                         command=["../rsync.sh", "-4", "--info=name", "--size-only", "--delay-updates",
 921                                  WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-a", "dl/", "%s/" %(rsync_src_url)],
 922                         env={'RSYNC_PASSWORD': rsync_src_key},
 923                         haltOnFailure = True,
 924                         logEnviron = False,
 925                         locks = NetLockUl,
 926                 ))
 927
 928         if False:
 929                 factory.addStep(ShellCommand(
 930                         name = "packageupload",
 931                         description = "Uploading package files",
 932                         command=["../rsync.sh", "-4", "--info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 933                         env={'RSYNC_PASSWORD': rsync_bin_key},
 934                         haltOnFailure = False,
 935                         logEnviron = False,
 936                         locks = NetLockUl,
 937                 ))
 938
 939         # logs
 940         if False:
 941                 factory.addStep(ShellCommand(
 942                         name = "upload",
 943                         description = "Uploading logs",
 944                         command=["../rsync.sh", "-4", "-info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 945                         env={'RSYNC_PASSWORD': rsync_bin_key},
 946                         haltOnFailure = False,
 947                         alwaysRun = True,
 948                         logEnviron = False,
 949                         locks = NetLockUl,
 950                 ))
 951
 952         factory.addStep(ShellCommand(
 953                 name = "df",
 954                 description = "Reporting disk usage",
 955                 command=["df", "-h", "."],
 956                 env={'LC_ALL': 'C'},
 957                 haltOnFailure = False,
 958                 alwaysRun = True
 959         ))
 960
 961         factory.addStep(ShellCommand(
 962                 name = "ccachestat",
 963                 description = "Reporting ccache stats",
 964                 command=["ccache", "-s"],
 965                 env = MakeEnv(overrides={ 'PATH': ["./staging_dir/host/bin", "${PATH}"] }),
 966                 want_stderr = False,
 967                 haltOnFailure = False,
 968                 flunkOnFailure = False,
 969                 warnOnFailure = False,
 970                 alwaysRun = True,
 971         ))
 972
 973         from buildbot.config import BuilderConfig
 974
 975         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))
 976
 977
 978 ####### STATUS TARGETS
 979
 980 # 'status' is a list of Status Targets. The results of each build will be
 981 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 982 # including web pages, email senders, and IRC bots.
 983
 984 c['status'] = []
 985
 986 from buildbot.status import html
 987 from buildbot.status.web import authz, auth
 988
 989 if ini.has_option("status", "bind"):
 990         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 991                 authz_cfg=authz.Authz(
 992                         # change any of these to True to enable; see the manual for more
 993                         # options
 994                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 995                         gracefulShutdown = 'auth',
 996                         forceBuild = 'auth', # use this to test your slave once it is set up
 997                         forceAllBuilds = 'auth',
 998                         pingBuilder = False,
 999                         stopBuild = 'auth',
1000                         stopAllBuilds = 'auth',
1001                         cancelPendingBuild = 'auth',
1002                 )
1003                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
1004         else:
1005                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
1006
1007
1008 from buildbot.status import words
1009
1010 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
1011         irc_host = ini.get("irc", "host")
1012         irc_port = 6667
1013         irc_chan = ini.get("irc", "channel")
1014         irc_nick = ini.get("irc", "nickname")
1015         irc_pass = None
1016
1017         if ini.has_option("irc", "port"):
1018                 irc_port = ini.getint("irc", "port")
1019
1020         if ini.has_option("irc", "password"):
1021                 irc_pass = ini.get("irc", "password")
1022
1023         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
1024                         channels = [{ "channel": irc_chan }],
1025                         notify_events = {
1026                           'exception': 1,
1027                           'successToFailure': 1,
1028                           'failureToSuccess': 1
1029                         }
1030         )
1031
1032         c['status'].append(irc)
1033
1034 ####### DB URL
1035
1036 c['db'] = {
1037         # This specifies what database buildbot uses to store its state.  You can leave
1038         # this at its default for all but the largest installations.
1039         'db_url' : "sqlite:///state.sqlite",
1040 }