phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### PROJECT IDENTITY
  22
  23 # the 'title' string will appear at the top of this buildbot
  24 # installation's html.WebStatus home page (linked to the
  25 # 'titleURL') and is embedded in the title of the waterfall HTML page.
  26
  27 c['title'] = ini.get("general", "title")
  28 c['titleURL'] = ini.get("general", "title_url")
  29
  30 # the 'buildbotURL' string should point to the location where the buildbot's
  31 # internal web server (usually the html.WebStatus page) is visible. This
  32 # typically uses the port number set in the Waterfall 'status' entry, but
  33 # with an externally-visible host name which the buildbot cannot figure out
  34 # without some help.
  35
  36 c['buildbotURL'] = ini.get("general", "buildbot_url")
  37
  38 ####### BUILDSLAVES
  39
  40 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  41 # a BuildSlave object, specifying a unique slave name and password.  The same
  42 # slave name and password must be configured on the slave.
  43 from buildbot.buildslave import BuildSlave
  44
  45 slave_port = 9989
  46
  47 if ini.has_option("general", "port"):
  48         slave_port = ini.getint("general", "port")
  49
  50 c['slaves'] = []
  51 max_builds = dict()
  52 do_cleanup = dict()
  53 NetLocks = dict()
  54
  55 for section in ini.sections():
  56         if section.startswith("slave "):
  57                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  58                         sl_props = { 'dl_lock':None, 'ul_lock':None, }
  59                         name = ini.get(section, "name")
  60                         password = ini.get(section, "password")
  61                         max_builds[name] = 1
  62                         do_cleanup[name] = False
  63                         if ini.has_option(section, "builds"):
  64                                 max_builds[name] = ini.getint(section, "builds")
  65                         if ini.has_option(section, "cleanup"):
  66                                 do_cleanup[name] = ini.getboolean(section, "cleanup")
  67                         if ini.has_option(section, "dl_lock"):
  68                                 lockname = ini.get(section, "dl_lock")
  69                                 sl_props['dl_lock'] = lockname
  70                                 if lockname not in NetLocks:
  71                                         NetLocks[lockname] = locks.MasterLock(lockname)
  72                         if ini.has_option(section, "ul_lock"):
  73                                 lockname = ini.get(section, "dl_lock")
  74                                 sl_props['ul_lock'] = lockname
  75                                 if lockname not in NetLocks:
  76                                         NetLocks[lockname] = locks.MasterLock(lockname)
  77                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name], properties = sl_props))
  78
  79 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  80 # This must match the value configured into the buildslaves (with their
  81 # --master option)
  82 c['slavePortnum'] = slave_port
  83
  84 # coalesce builds
  85 c['mergeRequests'] = True
  86
  87 # Reduce amount of backlog data
  88 c['buildHorizon'] = 30
  89 c['logHorizon'] = 20
  90
  91 ####### CHANGESOURCES
  92
  93 home_dir = os.path.abspath(ini.get("general", "homedir"))
  94 tree_expire = 0
  95 other_builds = 0
  96 cc_version = None
  97
  98 cc_command = "gcc"
  99 cxx_command = "g++"
 100
 101 if ini.has_option("general", "expire"):
 102         tree_expire = ini.getint("general", "expire")
 103
 104 if ini.has_option("general", "other_builds"):
 105         other_builds = ini.getint("general", "other_builds")
 106
 107 if ini.has_option("general", "cc_version"):
 108         cc_version = ini.get("general", "cc_version").split()
 109         if len(cc_version) == 1:
 110                 cc_version = ["eq", cc_version[0]]
 111
 112 repo_url = ini.get("repo", "url")
 113 repo_branch = "master"
 114
 115 if ini.has_option("repo", "branch"):
 116         repo_branch = ini.get("repo", "branch")
 117
 118 rsync_bin_url = ini.get("rsync", "binary_url")
 119 rsync_bin_key = ini.get("rsync", "binary_password")
 120
 121 rsync_src_url = None
 122 rsync_src_key = None
 123
 124 if ini.has_option("rsync", "source_url"):
 125         rsync_src_url = ini.get("rsync", "source_url")
 126         rsync_src_key = ini.get("rsync", "source_password")
 127
 128 gpg_home = "~/.gnupg"
 129 gpg_keyid = None
 130 gpg_comment = "Unattended build signature"
 131 gpg_passfile = "/dev/null"
 132
 133 if ini.has_option("gpg", "home"):
 134         gpg_home = ini.get("gpg", "home")
 135
 136 if ini.has_option("gpg", "keyid"):
 137         gpg_keyid = ini.get("gpg", "keyid")
 138
 139 if ini.has_option("gpg", "comment"):
 140         gpg_comment = ini.get("gpg", "comment")
 141
 142 if ini.has_option("gpg", "passfile"):
 143         gpg_passfile = ini.get("gpg", "passfile")
 144
 145 enable_kmod_archive = True
 146
 147
 148 # find targets
 149 targets = [ ]
 150
 151 if not os.path.isdir(home_dir+'/source.git'):
 152         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 153 else:
 154         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 155
 156 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 157         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 158
 159 while True:
 160         line = findtargets.stdout.readline()
 161         if not line:
 162                 break
 163         ta = line.strip().split(' ')
 164         targets.append(ta[0])
 165
 166
 167 # the 'change_source' setting tells the buildmaster how it should find out
 168 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 169
 170 from buildbot.changes.gitpoller import GitPoller
 171 c['change_source'] = []
 172 c['change_source'].append(GitPoller(
 173         repo_url,
 174         workdir=home_dir+'/work.git', branch=repo_branch,
 175         pollinterval=300))
 176
 177 ####### SCHEDULERS
 178
 179 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 180 # case, just kick off a 'basebuild' build
 181
 182 from buildbot.schedulers.basic import SingleBranchScheduler
 183 from buildbot.schedulers.forcesched import ForceScheduler
 184 from buildbot.changes import filter
 185 c['schedulers'] = []
 186 c['schedulers'].append(SingleBranchScheduler(
 187         name="all",
 188         change_filter=filter.ChangeFilter(branch=repo_branch),
 189         treeStableTimer=60,
 190         builderNames=targets))
 191
 192 c['schedulers'].append(ForceScheduler(
 193         name="force",
 194         builderNames=targets))
 195
 196 ####### BUILDERS
 197
 198 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 199 # what steps, and which slaves can execute them.  Note that any particular build will
 200 # only take place on one slave.
 201
 202 from buildbot.process.factory import BuildFactory
 203 from buildbot.steps.source.git import Git
 204 from buildbot.steps.shell import ShellCommand
 205 from buildbot.steps.shell import SetProperty
 206 from buildbot.steps.transfer import FileUpload
 207 from buildbot.steps.transfer import FileDownload
 208 from buildbot.steps.master import MasterShellCommand
 209 from buildbot.process.properties import WithProperties
 210 from buildbot.process import properties
 211
 212
 213 CleanTargetMap = [
 214         [ "tools",      "tools/clean"                   ],
 215         [ "chain",      "toolchain/clean"               ],
 216         [ "linux",      "target/linux/clean"    ],
 217         [ "dir",        "dirclean"                              ],
 218         [ "dist",       "distclean"                             ]
 219 ]
 220
 221 def IsMakeCleanRequested(pattern):
 222         def CheckCleanProperty(step):
 223                 val = step.getProperty("clean")
 224                 if val and re.match(pattern, val):
 225                         return True
 226                 else:
 227                         return False
 228
 229         return CheckCleanProperty
 230
 231 def IsCleanupRequested(step):
 232         val = step.getProperty("slavename")
 233         if val and do_cleanup[val]:
 234                 return True
 235         else:
 236                 return False
 237
 238 def IsExpireRequested(step):
 239         return not IsCleanupRequested(step)
 240
 241 def IsTaggingRequested(step):
 242         val = step.getProperty("tag")
 243         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
 244                 return True
 245         else:
 246                 return False
 247
 248 def IsNoTaggingRequested(step):
 249         return not IsTaggingRequested(step)
 250
 251 def IsNoMasterBuild(step):
 252         return repo_branch != "master"
 253
 254 def IsCleanupConfigured(step):
 255         slave = step.getProperty("slavename")
 256         if slave and slave in do_cleanup:
 257                 return do_cleanup[slave] > 0
 258         else:
 259                 return False
 260
 261 def GetBaseVersion(props):
 262         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 263                 return repo_branch.split('-')[1]
 264         else:
 265                 return "master"
 266
 267 def GetVersionPrefix(props):
 268         basever = GetBaseVersion(props)
 269         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 270                 return "%s/" % props["tag"]
 271         elif basever != "master":
 272                 return "%s-SNAPSHOT/" % basever
 273         else:
 274                 return ""
 275
 276 def GetNumJobs(props):
 277         if props.hasProperty("slavename") and props.hasProperty("nproc"):
 278                 return (int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds))
 279         else:
 280                 return 1
 281
 282 def GetCC(props):
 283         if props.hasProperty("cc_command"):
 284                 return props["cc_command"]
 285         else:
 286                 return "gcc"
 287
 288 def GetCXX(props):
 289         if props.hasProperty("cxx_command"):
 290                 return props["cxx_command"]
 291         else:
 292                 return "g++"
 293
 294 def GetCwd(props):
 295         if props.hasProperty("builddir"):
 296                 return props["builddir"]
 297         elif props.hasProperty("workdir"):
 298                 return props["workdir"]
 299         else:
 300                 return "/"
 301
 302 def GetNextBuild(builder, requests):
 303         for r in requests:
 304                 if r.properties and r.properties.hasProperty("tag"):
 305                         return r
 306         return requests[0]
 307
 308 def MakeEnv(overrides=None):
 309         env = {
 310                 'CC': WithProperties("%(cc)s", cc=GetCC),
 311                 'CXX': WithProperties("%(cxx)s", cxx=GetCXX),
 312                 'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)
 313         }
 314         if overrides is not None:
 315                 env.update(overrides)
 316         return env
 317
 318 @properties.renderer
 319 def NetLockDl(props):
 320         lock = None
 321         if props.hasProperty("dl_lock"):
 322                 lock = NetLocks[props["dl_lock"]]
 323         if lock is not None:
 324                 return [lock.access('exclusive')]
 325         else:
 326                 return []
 327
 328 @properties.renderer
 329 def NetLockUl(props):
 330         lock = None
 331         if props.hasProperty("ul_lock"):
 332                 lock = NetLocks[props["ul_lock"]]
 333         if lock is not None:
 334                 return [lock.access('exclusive')]
 335         else:
 336                 return []
 337
 338 c['builders'] = []
 339
 340 dlLock = locks.SlaveLock("slave_dl")
 341
 342 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 343         checkBuiltin() {
 344                 local symbol op path file;
 345                 for file in $CHANGED_FILES; do
 346                         case "$file" in
 347                                 package/*/*) : ;;
 348                                 *) return 0 ;;
 349                         esac;
 350                 done;
 351                 while read symbol op path; do
 352                         case "$symbol" in package-*)
 353                                 symbol="${symbol##*(}";
 354                                 symbol="${symbol%)}";
 355                                 for file in $CHANGED_FILES; do
 356                                         case "$file" in "package/$path/"*)
 357                                                 grep -qsx "$symbol=y" .config && return 0
 358                                         ;; esac;
 359                                 done;
 360                         esac;
 361                 done < tmp/.packagedeps;
 362                 return 1;
 363         }
 364 """).strip()
 365
 366
 367 class IfBuiltinShellCommand(ShellCommand):
 368         def _quote(self, str):
 369                 if re.search("[^a-zA-Z0-9/_.-]", str):
 370                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 371                 return str
 372
 373         def setCommand(self, command):
 374                 if not isinstance(command, (str, unicode)):
 375                         command = ' '.join(map(self._quote, command))
 376                 self.command = [
 377                         '/bin/sh', '-c',
 378                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 379                 ]
 380
 381         def setupEnvironment(self, cmd):
 382                 slaveEnv = self.slaveEnvironment
 383                 if slaveEnv is None:
 384                         slaveEnv = { }
 385                 changedFiles = { }
 386                 for request in self.build.requests:
 387                         for source in request.sources:
 388                                 for change in source.changes:
 389                                         for file in change.files:
 390                                                 changedFiles[file] = True
 391                 fullSlaveEnv = slaveEnv.copy()
 392                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 393                 cmd.args['env'] = fullSlaveEnv
 394
 395 slaveNames = [ ]
 396
 397 for slave in c['slaves']:
 398         slaveNames.append(slave.slavename)
 399
 400 for target in targets:
 401         ts = target.split('/')
 402
 403         factory = BuildFactory()
 404
 405         # find number of cores
 406         factory.addStep(SetProperty(
 407                 name = "nproc",
 408                 property = "nproc",
 409                 description = "Finding number of CPUs",
 410                 command = ["nproc"]))
 411
 412         # find gcc and g++ compilers
 413         if cc_version is not None:
 414                 factory.addStep(FileDownload(
 415                         mastersrc = "findbin.pl",
 416                         slavedest = "../findbin.pl",
 417                         mode = 0755))
 418
 419                 factory.addStep(SetProperty(
 420                         name = "gcc",
 421                         property = "cc_command",
 422                         description = "Finding gcc command",
 423                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 424                         haltOnFailure = True))
 425
 426                 factory.addStep(SetProperty(
 427                         name = "g++",
 428                         property = "cxx_command",
 429                         description = "Finding g++ command",
 430                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 431                         haltOnFailure = True))
 432
 433         # expire tree if needed
 434         if tree_expire > 0:
 435                 factory.addStep(FileDownload(
 436                         doStepIf = IsExpireRequested,
 437                         mastersrc = "expire.sh",
 438                         slavedest = "../expire.sh",
 439                         mode = 0755))
 440
 441                 factory.addStep(ShellCommand(
 442                         name = "expire",
 443                         description = "Checking for build tree expiry",
 444                         command = ["./expire.sh", str(tree_expire)],
 445                         workdir = ".",
 446                         haltOnFailure = True,
 447                         doStepIf = IsExpireRequested,
 448                         timeout = 2400))
 449
 450         # cleanup.sh if needed
 451         factory.addStep(FileDownload(
 452                 mastersrc = "cleanup.sh",
 453                 slavedest = "../cleanup.sh",
 454                 mode = 0755,
 455                 doStepIf = IsCleanupRequested))
 456
 457         factory.addStep(ShellCommand(
 458                 name = "cleanold",
 459                 description = "Cleaning previous builds",
 460                 command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 461                 workdir = ".",
 462                 haltOnFailure = True,
 463                 doStepIf = IsCleanupRequested,
 464                 timeout = 2400))
 465
 466         factory.addStep(ShellCommand(
 467                 name = "cleanup",
 468                 description = "Cleaning work area",
 469                 command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 470                 workdir = ".",
 471                 haltOnFailure = True,
 472                 doStepIf = IsCleanupRequested,
 473                 timeout = 2400))
 474
 475         # user-requested clean targets
 476         for tuple in CleanTargetMap:
 477                 factory.addStep(ShellCommand(
 478                         name = tuple[1],
 479                         description = 'User-requested "make %s"' % tuple[1],
 480                         command = ["make", tuple[1], "V=s"],
 481                         env = MakeEnv(),
 482                         doStepIf = IsMakeCleanRequested(tuple[0])
 483                 ))
 484
 485         # switch to branch
 486         factory.addStep(ShellCommand(
 487                 name = "switchbranch",
 488                 description = "Checking out Git branch",
 489                 command = "if [ -d .git ]; then git fetch && git checkout '%s'; else exit 0; fi" % repo_branch,
 490                 haltOnFailure = True,
 491                 doStepIf = IsNoTaggingRequested,
 492                 locks = NetLockDl,
 493         ))
 494
 495         # check out the source
 496         factory.addStep(Git(
 497                 repourl = repo_url,
 498                 branch = repo_branch,
 499                 mode = 'incremental',
 500                 method = 'clean',
 501                 locks = NetLockDl,
 502         ))
 503
 504         # update remote refs
 505         factory.addStep(ShellCommand(
 506                 name = "fetchrefs",
 507                 description = "Fetching Git remote refs",
 508                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 509                 haltOnFailure = True
 510         ))
 511
 512         # fetch tags
 513         factory.addStep(ShellCommand(
 514                 name = "fetchtag",
 515                 description = "Fetching Git tags",
 516                 command = ["git", "fetch", "--tags", "--", repo_url],
 517                 haltOnFailure = True,
 518                 doStepIf = IsTaggingRequested,
 519                 locks = NetLockDl,
 520         ))
 521
 522         # switch to tag
 523         factory.addStep(ShellCommand(
 524                 name = "switchtag",
 525                 description = "Checking out Git tag",
 526                 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
 527                 haltOnFailure = True,
 528                 doStepIf = IsTaggingRequested
 529         ))
 530
 531         factory.addStep(ShellCommand(
 532                 name = "rmtmp",
 533                 description = "Remove tmp folder",
 534                 command=["rm", "-rf", "tmp/"]))
 535
 536         # feed
 537 #       factory.addStep(ShellCommand(
 538 #               name = "feedsconf",
 539 #               description = "Copy the feeds.conf",
 540 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 541
 542         # feed
 543         factory.addStep(ShellCommand(
 544                 name = "rmfeedlinks",
 545                 description = "Remove feed symlinks",
 546                 command=["rm", "-rf", "package/feeds/"]))
 547
 548         # feed
 549         factory.addStep(ShellCommand(
 550                 name = "updatefeeds",
 551                 description = "Updating feeds",
 552                 command=["./scripts/feeds", "update"],
 553                 env = MakeEnv(),
 554                 locks = NetLockDl,
 555         ))
 556
 557         # feed
 558         factory.addStep(ShellCommand(
 559                 name = "installfeeds",
 560                 description = "Installing feeds",
 561                 command=["./scripts/feeds", "install", "-a"],
 562                 env = MakeEnv()))
 563
 564         # seed config
 565         factory.addStep(FileDownload(
 566                 mastersrc = "config.seed",
 567                 slavedest = ".config",
 568                 mode = 0644
 569         ))
 570
 571         # configure
 572         factory.addStep(ShellCommand(
 573                 name = "newconfig",
 574                 description = "Seeding .config",
 575                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 576         ))
 577
 578         factory.addStep(ShellCommand(
 579                 name = "delbin",
 580                 description = "Removing output directory",
 581                 command = ["rm", "-rf", "bin/"]
 582         ))
 583
 584         factory.addStep(ShellCommand(
 585                 name = "defconfig",
 586                 description = "Populating .config",
 587                 command = ["make", "defconfig"],
 588                 env = MakeEnv()
 589         ))
 590
 591         # check arch
 592         factory.addStep(ShellCommand(
 593                 name = "checkarch",
 594                 description = "Checking architecture",
 595                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 596                 logEnviron = False,
 597                 want_stdout = False,
 598                 want_stderr = False,
 599                 haltOnFailure = True
 600         ))
 601
 602         # find libc suffix
 603         factory.addStep(SetProperty(
 604                 name = "libc",
 605                 property = "libc",
 606                 description = "Finding libc suffix",
 607                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 608
 609         # ccache helper
 610         factory.addStep(FileDownload(
 611                 mastersrc = "ccache.sh",
 612                 slavedest = "ccache.sh",
 613                 mode = 0755
 614         ))
 615
 616         # ccache prepare
 617         factory.addStep(ShellCommand(
 618                 name = "prepccache",
 619                 description = "Preparing ccache",
 620                 command = ["./ccache.sh"]
 621         ))
 622
 623         # install build key
 624         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 625         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 626
 627         # prepare dl
 628         factory.addStep(ShellCommand(
 629                 name = "dldir",
 630                 description = "Preparing dl/",
 631                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 632                 logEnviron = False,
 633                 want_stdout = False
 634         ))
 635
 636         # prepare tar
 637         factory.addStep(ShellCommand(
 638                 name = "dltar",
 639                 description = "Building and installing GNU tar",
 640                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
 641                 env = MakeEnv(),
 642                 haltOnFailure = True
 643         ))
 644
 645         # populate dl
 646         factory.addStep(ShellCommand(
 647                 name = "dlrun",
 648                 description = "Populating dl/",
 649                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
 650                 env = MakeEnv(),
 651                 logEnviron = False,
 652                 locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
 653         ))
 654
 655         factory.addStep(ShellCommand(
 656                 name = "cleanbase",
 657                 description = "Cleaning base-files",
 658                 command=["make", "package/base-files/clean", "V=s"]
 659         ))
 660
 661         # build
 662         factory.addStep(ShellCommand(
 663                 name = "tools",
 664                 description = "Building and installing tools",
 665                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
 666                 env = MakeEnv(),
 667                 haltOnFailure = True
 668         ))
 669
 670         factory.addStep(ShellCommand(
 671                 name = "toolchain",
 672                 description = "Building and installing toolchain",
 673                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
 674                 env = MakeEnv(),
 675                 haltOnFailure = True
 676         ))
 677
 678         factory.addStep(ShellCommand(
 679                 name = "kmods",
 680                 description = "Building kmods",
 681                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 682                 env = MakeEnv(),
 683                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 684                 haltOnFailure = True
 685         ))
 686
 687         # find kernel version
 688         factory.addStep(SetProperty(
 689                 name = "kernelversion",
 690                 property = "kernelversion",
 691                 description = "Finding the effective Kernel version",
 692                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
 693                 env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) }
 694         ))
 695
 696         factory.addStep(ShellCommand(
 697                 name = "pkgclean",
 698                 description = "Cleaning up package build",
 699                 command=["make", "package/cleanup", "V=s"]
 700         ))
 701
 702         factory.addStep(ShellCommand(
 703                 name = "pkgbuild",
 704                 description = "Building packages",
 705                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 706                 env = MakeEnv(),
 707                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 708                 haltOnFailure = True
 709         ))
 710
 711         # factory.addStep(IfBuiltinShellCommand(
 712         factory.addStep(ShellCommand(
 713                 name = "pkginstall",
 714                 description = "Installing packages",
 715                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
 716                 env = MakeEnv(),
 717                 haltOnFailure = True
 718         ))
 719
 720         factory.addStep(ShellCommand(
 721                 name = "pkgindex",
 722                 description = "Indexing packages",
 723                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
 724                 env = MakeEnv(),
 725                 haltOnFailure = True
 726         ))
 727
 728         if enable_kmod_archive:
 729                 factory.addStep(ShellCommand(
 730                         name = "kmoddir",
 731                         description = "Creating kmod directory",
 732                         command=["mkdir", "-p", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s" %(ts[0], ts[1]))],
 733                         haltOnFailure = True
 734                 ))
 735
 736                 factory.addStep(ShellCommand(
 737                         name = "kmodprepare",
 738                         description = "Preparing kmod archive",
 739                         command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
 740                                  WithProperties("bin/targets/%s/%s%%(libc)s/packages/" %(ts[0], ts[1])),
 741                                  WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))],
 742                         haltOnFailure = True
 743                 ))
 744
 745                 factory.addStep(ShellCommand(
 746                         name = "kmodindex",
 747                         description = "Indexing kmod archive",
 748                         command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s",
 749                                  WithProperties("PACKAGE_SUBDIRS=bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))],
 750                         env = MakeEnv(),
 751                         haltOnFailure = True
 752                 ))
 753
 754                 # find rootfs staging directory
 755                 factory.addStep(SetProperty(
 756                         name = "stageroot",
 757                         property = "stageroot",
 758                         description = "Finding the rootfs staging directory",
 759                         command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"],
 760                         env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) }
 761                 ))
 762
 763                 factory.addStep(ShellCommand(
 764                         name = "filesdir",
 765                         description = "Creating file overlay directory",
 766                         command=["mkdir", "-p", "files/etc/opkg"],
 767                         haltOnFailure = True
 768                 ))
 769
 770                 factory.addStep(ShellCommand(
 771                         name = "kmodconfig",
 772                         description = "Embedding kmod repository configuration",
 773                         command=WithProperties("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(kernelversion)s#' " +
 774                                                "%(stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"),
 775                         haltOnFailure = True
 776                 ))
 777
 778         #factory.addStep(IfBuiltinShellCommand(
 779         factory.addStep(ShellCommand(
 780                 name = "images",
 781                 description = "Building and installing images",
 782                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
 783                 env = MakeEnv(),
 784                 haltOnFailure = True
 785         ))
 786
 787         factory.addStep(ShellCommand(
 788                 name = "diffconfig",
 789                 description = "Generating config.seed",
 790                 command=["make", "-j1", "diffconfig", "V=s"],
 791                 env = MakeEnv(),
 792                 haltOnFailure = True
 793         ))
 794
 795         factory.addStep(ShellCommand(
 796                 name = "checksums",
 797                 description = "Calculating checksums",
 798                 command=["make", "-j1", "checksum", "V=s"],
 799                 env = MakeEnv(),
 800                 haltOnFailure = True
 801         ))
 802
 803         # sign
 804         if gpg_keyid is not None:
 805                 factory.addStep(MasterShellCommand(
 806                         name = "signprepare",
 807                         description = "Preparing temporary signing directory",
 808                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 809                         haltOnFailure = True
 810                 ))
 811
 812                 factory.addStep(ShellCommand(
 813                         name = "signpack",
 814                         description = "Packing files to sign",
 815                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ bin/targets/%s/%s%%(libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1], ts[0], ts[1])),
 816                         haltOnFailure = True
 817                 ))
 818
 819                 factory.addStep(FileUpload(
 820                         slavesrc = "sign.tar.gz",
 821                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 822                         haltOnFailure = True
 823                 ))
 824
 825                 factory.addStep(MasterShellCommand(
 826                         name = "signfiles",
 827                         description = "Signing files",
 828                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 829                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 830                         haltOnFailure = True
 831                 ))
 832
 833                 factory.addStep(FileDownload(
 834                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 835                         slavedest = "sign.tar.gz",
 836                         haltOnFailure = True
 837                 ))
 838
 839                 factory.addStep(ShellCommand(
 840                         name = "signunpack",
 841                         description = "Unpacking signed files",
 842                         command = ["tar", "-xzf", "sign.tar.gz"],
 843                         haltOnFailure = True
 844                 ))
 845
 846         # upload
 847         factory.addStep(ShellCommand(
 848                 name = "dirprepare",
 849                 description = "Preparing upload directory structure",
 850                 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 851                 haltOnFailure = True
 852         ))
 853
 854         factory.addStep(ShellCommand(
 855                 name = "linkprepare",
 856                 description = "Preparing repository symlink",
 857                 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
 858                 doStepIf = IsNoMasterBuild,
 859                 haltOnFailure = True
 860         ))
 861
 862         if enable_kmod_archive:
 863                 factory.addStep(ShellCommand(
 864                         name = "kmoddirprepare",
 865                         description = "Preparing kmod archive upload directory",
 866                         command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 867                         haltOnFailure = True
 868                 ))
 869
 870         factory.addStep(ShellCommand(
 871                 name = "dirupload",
 872                 description = "Uploading directory structure",
 873                 command = ["rsync", "-4", "--info=name", "-az", "tmp/upload/", "%s/" %(rsync_bin_url)],
 874                 env={'RSYNC_PASSWORD': rsync_bin_key},
 875                 haltOnFailure = True,
 876                 logEnviron = False,
 877                 locks = NetLockUl,
 878         ))
 879
 880         factory.addStep(FileDownload(
 881                         mastersrc = "rsync.sh",
 882                         slavedest = "../rsync.sh",
 883                         mode = 0755))
 884
 885         factory.addStep(ShellCommand(
 886                 name = "targetupload",
 887                 description = "Uploading target files",
 888                 command=["../rsync.sh", "-4", "--info=name", "--exclude=/kmods/", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 889                          "-a", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
 890                          WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 891                 env={'RSYNC_PASSWORD': rsync_bin_key},
 892                 haltOnFailure = True,
 893                 logEnviron = False,
 894                 locks = NetLockUl,
 895         ))
 896
 897         if enable_kmod_archive:
 898                 factory.addStep(ShellCommand(
 899                         name = "kmodupload",
 900                         description = "Uploading kmod archive",
 901                         command=["../rsync.sh", "-4", "--info=name", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 902                                  "-a", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])),
 903                                  WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 904                         env={'RSYNC_PASSWORD': rsync_bin_key},
 905                         haltOnFailure = True,
 906                         logEnviron = False,
 907                         locks = NetLockUl,
 908                 ))
 909
 910         if rsync_src_url is not None:
 911                 factory.addStep(ShellCommand(
 912                         name = "sourceupload",
 913                         description = "Uploading source archives",
 914                         command=["../rsync.sh", "-4", "--info=name", "--size-only", "--delay-updates",
 915                                  WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-a", "dl/", "%s/" %(rsync_src_url)],
 916                         env={'RSYNC_PASSWORD': rsync_src_key},
 917                         haltOnFailure = True,
 918                         logEnviron = False,
 919                         locks = NetLockUl,
 920                 ))
 921
 922         if False:
 923                 factory.addStep(ShellCommand(
 924                         name = "packageupload",
 925                         description = "Uploading package files",
 926                         command=["../rsync.sh", "-4", "--info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 927                         env={'RSYNC_PASSWORD': rsync_bin_key},
 928                         haltOnFailure = False,
 929                         logEnviron = False,
 930                         locks = NetLockUl,
 931                 ))
 932
 933         # logs
 934         if False:
 935                 factory.addStep(ShellCommand(
 936                         name = "upload",
 937                         description = "Uploading logs",
 938                         command=["../rsync.sh", "-4", "-info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 939                         env={'RSYNC_PASSWORD': rsync_bin_key},
 940                         haltOnFailure = False,
 941                         alwaysRun = True,
 942                         logEnviron = False,
 943                         locks = NetLockUl,
 944                 ))
 945
 946         factory.addStep(ShellCommand(
 947                 name = "df",
 948                 description = "Reporting disk usage",
 949                 command=["df", "-h", "."],
 950                 env={'LC_ALL': 'C'},
 951                 haltOnFailure = False,
 952                 alwaysRun = True
 953         ))
 954
 955         from buildbot.config import BuilderConfig
 956
 957         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))
 958
 959
 960 ####### STATUS TARGETS
 961
 962 # 'status' is a list of Status Targets. The results of each build will be
 963 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 964 # including web pages, email senders, and IRC bots.
 965
 966 c['status'] = []
 967
 968 from buildbot.status import html
 969 from buildbot.status.web import authz, auth
 970
 971 if ini.has_option("status", "bind"):
 972         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 973                 authz_cfg=authz.Authz(
 974                         # change any of these to True to enable; see the manual for more
 975                         # options
 976                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 977                         gracefulShutdown = 'auth',
 978                         forceBuild = 'auth', # use this to test your slave once it is set up
 979                         forceAllBuilds = 'auth',
 980                         pingBuilder = False,
 981                         stopBuild = 'auth',
 982                         stopAllBuilds = 'auth',
 983                         cancelPendingBuild = 'auth',
 984                 )
 985                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 986         else:
 987                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 988
 989
 990 from buildbot.status import words
 991
 992 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 993         irc_host = ini.get("irc", "host")
 994         irc_port = 6667
 995         irc_chan = ini.get("irc", "channel")
 996         irc_nick = ini.get("irc", "nickname")
 997         irc_pass = None
 998
 999         if ini.has_option("irc", "port"):
1000                 irc_port = ini.getint("irc", "port")
1001
1002         if ini.has_option("irc", "password"):
1003                 irc_pass = ini.get("irc", "password")
1004
1005         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
1006                         channels = [{ "channel": irc_chan }],
1007                         notify_events = {
1008                           'exception': 1,
1009                           'successToFailure': 1,
1010                           'failureToSuccess': 1
1011                         }
1012         )
1013
1014         c['status'].append(irc)
1015
1016 ####### DB URL
1017
1018 c['db'] = {
1019         # This specifies what database buildbot uses to store its state.  You can leave
1020         # this at its default for all but the largest installations.
1021         'db_url' : "sqlite:///state.sqlite",
1022 }