phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### PROJECT IDENTITY
  22
  23 # the 'title' string will appear at the top of this buildbot
  24 # installation's html.WebStatus home page (linked to the
  25 # 'titleURL') and is embedded in the title of the waterfall HTML page.
  26
  27 c['title'] = ini.get("general", "title")
  28 c['titleURL'] = ini.get("general", "title_url")
  29
  30 # the 'buildbotURL' string should point to the location where the buildbot's
  31 # internal web server (usually the html.WebStatus page) is visible. This
  32 # typically uses the port number set in the Waterfall 'status' entry, but
  33 # with an externally-visible host name which the buildbot cannot figure out
  34 # without some help.
  35
  36 c['buildbotURL'] = ini.get("general", "buildbot_url")
  37
  38 ####### BUILDSLAVES
  39
  40 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  41 # a BuildSlave object, specifying a unique slave name and password.  The same
  42 # slave name and password must be configured on the slave.
  43 from buildbot.buildslave import BuildSlave
  44
  45 slave_port = 9989
  46
  47 if ini.has_option("general", "port"):
  48         slave_port = ini.getint("general", "port")
  49
  50 c['slaves'] = []
  51 max_builds = dict()
  52 do_cleanup = dict()
  53 NetLocks = dict()
  54
  55 for section in ini.sections():
  56         if section.startswith("slave "):
  57                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  58                         sl_props = { 'dl_lock':None, 'ul_lock':None, }
  59                         name = ini.get(section, "name")
  60                         password = ini.get(section, "password")
  61                         max_builds[name] = 1
  62                         do_cleanup[name] = False
  63                         if ini.has_option(section, "builds"):
  64                                 max_builds[name] = ini.getint(section, "builds")
  65                         if ini.has_option(section, "cleanup"):
  66                                 do_cleanup[name] = ini.getboolean(section, "cleanup")
  67                         if ini.has_option(section, "dl_lock"):
  68                                 lockname = ini.get(section, "dl_lock")
  69                                 sl_props['dl_lock'] = lockname
  70                                 if lockname not in NetLocks:
  71                                         NetLocks[lockname] = locks.MasterLock(lockname)
  72                         if ini.has_option(section, "ul_lock"):
  73                                 lockname = ini.get(section, "dl_lock")
  74                                 sl_props['ul_lock'] = lockname
  75                                 if lockname not in NetLocks:
  76                                         NetLocks[lockname] = locks.MasterLock(lockname)
  77                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name], properties = sl_props))
  78
  79 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  80 # This must match the value configured into the buildslaves (with their
  81 # --master option)
  82 c['slavePortnum'] = slave_port
  83
  84 # coalesce builds
  85 c['mergeRequests'] = True
  86
  87 # Reduce amount of backlog data
  88 c['buildHorizon'] = 30
  89 c['logHorizon'] = 20
  90
  91 ####### CHANGESOURCES
  92
  93 home_dir = os.path.abspath(ini.get("general", "homedir"))
  94 tree_expire = 0
  95 other_builds = 0
  96 cc_version = None
  97
  98 cc_command = "gcc"
  99 cxx_command = "g++"
 100
 101 if ini.has_option("general", "expire"):
 102         tree_expire = ini.getint("general", "expire")
 103
 104 if ini.has_option("general", "other_builds"):
 105         other_builds = ini.getint("general", "other_builds")
 106
 107 if ini.has_option("general", "cc_version"):
 108         cc_version = ini.get("general", "cc_version").split()
 109         if len(cc_version) == 1:
 110                 cc_version = ["eq", cc_version[0]]
 111
 112 repo_url = ini.get("repo", "url")
 113 repo_branch = "master"
 114
 115 if ini.has_option("repo", "branch"):
 116         repo_branch = ini.get("repo", "branch")
 117
 118 rsync_bin_url = ini.get("rsync", "binary_url")
 119 rsync_bin_key = ini.get("rsync", "binary_password")
 120
 121 rsync_src_url = None
 122 rsync_src_key = None
 123
 124 if ini.has_option("rsync", "source_url"):
 125         rsync_src_url = ini.get("rsync", "source_url")
 126         rsync_src_key = ini.get("rsync", "source_password")
 127
 128 gpg_home = "~/.gnupg"
 129 gpg_keyid = None
 130 gpg_comment = "Unattended build signature"
 131 gpg_passfile = "/dev/null"
 132
 133 if ini.has_option("gpg", "home"):
 134         gpg_home = ini.get("gpg", "home")
 135
 136 if ini.has_option("gpg", "keyid"):
 137         gpg_keyid = ini.get("gpg", "keyid")
 138
 139 if ini.has_option("gpg", "comment"):
 140         gpg_comment = ini.get("gpg", "comment")
 141
 142 if ini.has_option("gpg", "passfile"):
 143         gpg_passfile = ini.get("gpg", "passfile")
 144
 145 enable_kmod_archive = True
 146
 147
 148 # find targets
 149 targets = [ ]
 150
 151 if not os.path.isdir(home_dir+'/source.git'):
 152         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 153 else:
 154         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 155
 156 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 157         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 158
 159 while True:
 160         line = findtargets.stdout.readline()
 161         if not line:
 162                 break
 163         ta = line.strip().split(' ')
 164         targets.append(ta[0])
 165
 166
 167 # the 'change_source' setting tells the buildmaster how it should find out
 168 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 169
 170 from buildbot.changes.gitpoller import GitPoller
 171 c['change_source'] = []
 172 c['change_source'].append(GitPoller(
 173         repo_url,
 174         workdir=home_dir+'/work.git', branch=repo_branch,
 175         pollinterval=300))
 176
 177 ####### SCHEDULERS
 178
 179 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 180 # case, just kick off a 'basebuild' build
 181
 182 from buildbot.schedulers.basic import SingleBranchScheduler
 183 from buildbot.schedulers.forcesched import ForceScheduler
 184 from buildbot.changes import filter
 185 c['schedulers'] = []
 186 c['schedulers'].append(SingleBranchScheduler(
 187         name="all",
 188         change_filter=filter.ChangeFilter(branch=repo_branch),
 189         treeStableTimer=60,
 190         builderNames=targets))
 191
 192 c['schedulers'].append(ForceScheduler(
 193         name="force",
 194         builderNames=targets))
 195
 196 ####### BUILDERS
 197
 198 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 199 # what steps, and which slaves can execute them.  Note that any particular build will
 200 # only take place on one slave.
 201
 202 from buildbot.process.factory import BuildFactory
 203 from buildbot.steps.source.git import Git
 204 from buildbot.steps.shell import ShellCommand
 205 from buildbot.steps.shell import SetProperty
 206 from buildbot.steps.transfer import FileUpload
 207 from buildbot.steps.transfer import FileDownload
 208 from buildbot.steps.master import MasterShellCommand
 209 from buildbot.process.properties import WithProperties
 210 from buildbot.process import properties
 211
 212
 213 CleanTargetMap = [
 214         [ "tools",      "tools/clean"                   ],
 215         [ "chain",      "toolchain/clean"               ],
 216         [ "linux",      "target/linux/clean"    ],
 217         [ "dir",        "dirclean"                              ],
 218         [ "dist",       "distclean"                             ]
 219 ]
 220
 221 def IsMakeCleanRequested(pattern):
 222         def CheckCleanProperty(step):
 223                 val = step.getProperty("clean")
 224                 if val and re.match(pattern, val):
 225                         return True
 226                 else:
 227                         return False
 228
 229         return CheckCleanProperty
 230
 231 def IsCleanupRequested(step):
 232         val = step.getProperty("slavename")
 233         if val and do_cleanup[val]:
 234                 return True
 235         else:
 236                 return False
 237
 238 def IsExpireRequested(step):
 239         return not IsCleanupRequested(step)
 240
 241 def IsTaggingRequested(step):
 242         val = step.getProperty("tag")
 243         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
 244                 return True
 245         else:
 246                 return False
 247
 248 def IsNoTaggingRequested(step):
 249         return not IsTaggingRequested(step)
 250
 251 def IsNoMasterBuild(step):
 252         return repo_branch != "master"
 253
 254 def IsCleanupConfigured(step):
 255         slave = step.getProperty("slavename")
 256         if slave and slave in do_cleanup:
 257                 return do_cleanup[slave] > 0
 258         else:
 259                 return False
 260
 261 def GetBaseVersion(props):
 262         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 263                 return repo_branch.split('-')[1]
 264         else:
 265                 return "master"
 266
 267 def GetVersionPrefix(props):
 268         basever = GetBaseVersion(props)
 269         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 270                 return "%s/" % props["tag"]
 271         elif basever != "master":
 272                 return "%s-SNAPSHOT/" % basever
 273         else:
 274                 return ""
 275
 276 def GetNumJobs(props):
 277         if props.hasProperty("slavename") and props.hasProperty("nproc"):
 278                 return (int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds))
 279         else:
 280                 return 1
 281
 282 def GetCC(props):
 283         if props.hasProperty("cc_command"):
 284                 return props["cc_command"]
 285         else:
 286                 return "gcc"
 287
 288 def GetCXX(props):
 289         if props.hasProperty("cxx_command"):
 290                 return props["cxx_command"]
 291         else:
 292                 return "g++"
 293
 294 def GetCwd(props):
 295         if props.hasProperty("builddir"):
 296                 return props["builddir"]
 297         elif props.hasProperty("workdir"):
 298                 return props["workdir"]
 299         else:
 300                 return "/"
 301
 302 def GetNextBuild(builder, requests):
 303         for r in requests:
 304                 if r.properties and r.properties.hasProperty("tag"):
 305                         return r
 306         return requests[0]
 307
 308 def MakeEnv(overrides=None):
 309         env = {
 310                 'CC': WithProperties("%(cc)s", cc=GetCC),
 311                 'CXX': WithProperties("%(cxx)s", cxx=GetCXX),
 312                 'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)
 313         }
 314         if overrides is not None:
 315                 env.update(overrides)
 316         return env
 317
 318 @properties.renderer
 319 def NetLockDl(props):
 320         lock = None
 321         if props.hasProperty("dl_lock"):
 322                 lock = NetLocks[props["dl_lock"]]
 323         if lock is not None:
 324                 return [lock.access('exclusive')]
 325         else:
 326                 return []
 327
 328 @properties.renderer
 329 def NetLockUl(props):
 330         lock = None
 331         if props.hasProperty("ul_lock"):
 332                 lock = NetLocks[props["ul_lock"]]
 333         if lock is not None:
 334                 return [lock.access('exclusive')]
 335         else:
 336                 return []
 337
 338 c['builders'] = []
 339
 340 dlLock = locks.SlaveLock("slave_dl")
 341
 342 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 343         checkBuiltin() {
 344                 local symbol op path file;
 345                 for file in $CHANGED_FILES; do
 346                         case "$file" in
 347                                 package/*/*) : ;;
 348                                 *) return 0 ;;
 349                         esac;
 350                 done;
 351                 while read symbol op path; do
 352                         case "$symbol" in package-*)
 353                                 symbol="${symbol##*(}";
 354                                 symbol="${symbol%)}";
 355                                 for file in $CHANGED_FILES; do
 356                                         case "$file" in "package/$path/"*)
 357                                                 grep -qsx "$symbol=y" .config && return 0
 358                                         ;; esac;
 359                                 done;
 360                         esac;
 361                 done < tmp/.packagedeps;
 362                 return 1;
 363         }
 364 """).strip()
 365
 366
 367 class IfBuiltinShellCommand(ShellCommand):
 368         def _quote(self, str):
 369                 if re.search("[^a-zA-Z0-9/_.-]", str):
 370                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 371                 return str
 372
 373         def setCommand(self, command):
 374                 if not isinstance(command, (str, unicode)):
 375                         command = ' '.join(map(self._quote, command))
 376                 self.command = [
 377                         '/bin/sh', '-c',
 378                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 379                 ]
 380
 381         def setupEnvironment(self, cmd):
 382                 slaveEnv = self.slaveEnvironment
 383                 if slaveEnv is None:
 384                         slaveEnv = { }
 385                 changedFiles = { }
 386                 for request in self.build.requests:
 387                         for source in request.sources:
 388                                 for change in source.changes:
 389                                         for file in change.files:
 390                                                 changedFiles[file] = True
 391                 fullSlaveEnv = slaveEnv.copy()
 392                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 393                 cmd.args['env'] = fullSlaveEnv
 394
 395 slaveNames = [ ]
 396
 397 for slave in c['slaves']:
 398         slaveNames.append(slave.slavename)
 399
 400 for target in targets:
 401         ts = target.split('/')
 402
 403         factory = BuildFactory()
 404
 405         # find number of cores
 406         factory.addStep(SetProperty(
 407                 name = "nproc",
 408                 property = "nproc",
 409                 description = "Finding number of CPUs",
 410                 command = ["nproc"]))
 411
 412         # find gcc and g++ compilers
 413         if cc_version is not None:
 414                 factory.addStep(FileDownload(
 415                         mastersrc = "findbin.pl",
 416                         slavedest = "../findbin.pl",
 417                         mode = 0755))
 418
 419                 factory.addStep(SetProperty(
 420                         name = "gcc",
 421                         property = "cc_command",
 422                         description = "Finding gcc command",
 423                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 424                         haltOnFailure = True))
 425
 426                 factory.addStep(SetProperty(
 427                         name = "g++",
 428                         property = "cxx_command",
 429                         description = "Finding g++ command",
 430                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 431                         haltOnFailure = True))
 432
 433         # expire tree if needed
 434         if tree_expire > 0:
 435                 factory.addStep(FileDownload(
 436                         doStepIf = IsExpireRequested,
 437                         mastersrc = "expire.sh",
 438                         slavedest = "../expire.sh",
 439                         mode = 0755))
 440
 441                 factory.addStep(ShellCommand(
 442                         name = "expire",
 443                         description = "Checking for build tree expiry",
 444                         command = ["./expire.sh", str(tree_expire)],
 445                         workdir = ".",
 446                         haltOnFailure = True,
 447                         doStepIf = IsExpireRequested,
 448                         timeout = 2400))
 449
 450         # cleanup.sh if needed
 451         factory.addStep(FileDownload(
 452                 mastersrc = "cleanup.sh",
 453                 slavedest = "../cleanup.sh",
 454                 mode = 0755,
 455                 doStepIf = IsCleanupRequested))
 456
 457         factory.addStep(ShellCommand(
 458                 name = "cleanold",
 459                 description = "Cleaning previous builds",
 460                 command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 461                 workdir = ".",
 462                 haltOnFailure = True,
 463                 doStepIf = IsCleanupRequested,
 464                 timeout = 2400))
 465
 466         factory.addStep(ShellCommand(
 467                 name = "cleanup",
 468                 description = "Cleaning work area",
 469                 command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 470                 workdir = ".",
 471                 haltOnFailure = True,
 472                 doStepIf = IsCleanupRequested,
 473                 timeout = 2400))
 474
 475         # user-requested clean targets
 476         for tuple in CleanTargetMap:
 477                 factory.addStep(ShellCommand(
 478                         name = tuple[1],
 479                         description = 'User-requested "make %s"' % tuple[1],
 480                         command = ["make", tuple[1], "V=s"],
 481                         env = MakeEnv(),
 482                         doStepIf = IsMakeCleanRequested(tuple[0])
 483                 ))
 484
 485         # check out the source
 486         # Git() runs:
 487           # if repo doesn't exist: 'git clone repourl'
 488           # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
 489           # 'git fetch -t repourl branch; git reset --hard revision'
 490         factory.addStep(Git(
 491                 repourl = repo_url,
 492                 branch = repo_branch,
 493                 mode = 'full',
 494                 method = 'clean',
 495                 locks = NetLockDl,
 496                 haltOnFailure = True,
 497         ))
 498
 499         # update remote refs
 500         factory.addStep(ShellCommand(
 501                 name = "fetchrefs",
 502                 description = "Fetching Git remote refs",
 503                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 504                 haltOnFailure = True
 505         ))
 506
 507         # switch to tag
 508         factory.addStep(ShellCommand(
 509                 name = "switchtag",
 510                 description = "Checking out Git tag",
 511                 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
 512                 haltOnFailure = True,
 513                 doStepIf = IsTaggingRequested
 514         ))
 515
 516         factory.addStep(ShellCommand(
 517                 name = "rmtmp",
 518                 description = "Remove tmp folder",
 519                 command=["rm", "-rf", "tmp/"]))
 520
 521         # feed
 522 #       factory.addStep(ShellCommand(
 523 #               name = "feedsconf",
 524 #               description = "Copy the feeds.conf",
 525 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 526
 527         # feed
 528         factory.addStep(ShellCommand(
 529                 name = "rmfeedlinks",
 530                 description = "Remove feed symlinks",
 531                 command=["rm", "-rf", "package/feeds/"]))
 532
 533         # feed
 534         factory.addStep(ShellCommand(
 535                 name = "updatefeeds",
 536                 description = "Updating feeds",
 537                 command=["./scripts/feeds", "update"],
 538                 env = MakeEnv(),
 539                 locks = NetLockDl,
 540         ))
 541
 542         # feed
 543         factory.addStep(ShellCommand(
 544                 name = "installfeeds",
 545                 description = "Installing feeds",
 546                 command=["./scripts/feeds", "install", "-a"],
 547                 env = MakeEnv()))
 548
 549         # seed config
 550         factory.addStep(FileDownload(
 551                 mastersrc = "config.seed",
 552                 slavedest = ".config",
 553                 mode = 0644
 554         ))
 555
 556         # configure
 557         factory.addStep(ShellCommand(
 558                 name = "newconfig",
 559                 description = "Seeding .config",
 560                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 561         ))
 562
 563         factory.addStep(ShellCommand(
 564                 name = "delbin",
 565                 description = "Removing output directory",
 566                 command = ["rm", "-rf", "bin/"]
 567         ))
 568
 569         factory.addStep(ShellCommand(
 570                 name = "defconfig",
 571                 description = "Populating .config",
 572                 command = ["make", "defconfig"],
 573                 env = MakeEnv()
 574         ))
 575
 576         # check arch
 577         factory.addStep(ShellCommand(
 578                 name = "checkarch",
 579                 description = "Checking architecture",
 580                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 581                 logEnviron = False,
 582                 want_stdout = False,
 583                 want_stderr = False,
 584                 haltOnFailure = True
 585         ))
 586
 587         # find libc suffix
 588         factory.addStep(SetProperty(
 589                 name = "libc",
 590                 property = "libc",
 591                 description = "Finding libc suffix",
 592                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 593
 594         # ccache helper
 595         factory.addStep(FileDownload(
 596                 mastersrc = "ccache.sh",
 597                 slavedest = "ccache.sh",
 598                 mode = 0755
 599         ))
 600
 601         # ccache prepare
 602         factory.addStep(ShellCommand(
 603                 name = "prepccache",
 604                 description = "Preparing ccache",
 605                 command = ["./ccache.sh"]
 606         ))
 607
 608         # install build key
 609         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 610         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 611
 612         # prepare dl
 613         factory.addStep(ShellCommand(
 614                 name = "dldir",
 615                 description = "Preparing dl/",
 616                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 617                 logEnviron = False,
 618                 want_stdout = False
 619         ))
 620
 621         # prepare tar
 622         factory.addStep(ShellCommand(
 623                 name = "dltar",
 624                 description = "Building and installing GNU tar",
 625                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
 626                 env = MakeEnv(),
 627                 haltOnFailure = True
 628         ))
 629
 630         # populate dl
 631         factory.addStep(ShellCommand(
 632                 name = "dlrun",
 633                 description = "Populating dl/",
 634                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
 635                 env = MakeEnv(),
 636                 logEnviron = False,
 637                 locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
 638         ))
 639
 640         factory.addStep(ShellCommand(
 641                 name = "cleanbase",
 642                 description = "Cleaning base-files",
 643                 command=["make", "package/base-files/clean", "V=s"]
 644         ))
 645
 646         # build
 647         factory.addStep(ShellCommand(
 648                 name = "tools",
 649                 description = "Building and installing tools",
 650                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
 651                 env = MakeEnv(),
 652                 haltOnFailure = True
 653         ))
 654
 655         factory.addStep(ShellCommand(
 656                 name = "toolchain",
 657                 description = "Building and installing toolchain",
 658                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
 659                 env = MakeEnv(),
 660                 haltOnFailure = True
 661         ))
 662
 663         factory.addStep(ShellCommand(
 664                 name = "kmods",
 665                 description = "Building kmods",
 666                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 667                 env = MakeEnv(),
 668                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 669                 haltOnFailure = True
 670         ))
 671
 672         # find kernel version
 673         factory.addStep(SetProperty(
 674                 name = "kernelversion",
 675                 property = "kernelversion",
 676                 description = "Finding the effective Kernel version",
 677                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
 678                 env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) }
 679         ))
 680
 681         factory.addStep(ShellCommand(
 682                 name = "pkgclean",
 683                 description = "Cleaning up package build",
 684                 command=["make", "package/cleanup", "V=s"]
 685         ))
 686
 687         factory.addStep(ShellCommand(
 688                 name = "pkgbuild",
 689                 description = "Building packages",
 690                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 691                 env = MakeEnv(),
 692                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 693                 haltOnFailure = True
 694         ))
 695
 696         # factory.addStep(IfBuiltinShellCommand(
 697         factory.addStep(ShellCommand(
 698                 name = "pkginstall",
 699                 description = "Installing packages",
 700                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
 701                 env = MakeEnv(),
 702                 haltOnFailure = True
 703         ))
 704
 705         factory.addStep(ShellCommand(
 706                 name = "pkgindex",
 707                 description = "Indexing packages",
 708                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
 709                 env = MakeEnv(),
 710                 haltOnFailure = True
 711         ))
 712
 713         if enable_kmod_archive:
 714                 factory.addStep(ShellCommand(
 715                         name = "kmoddir",
 716                         description = "Creating kmod directory",
 717                         command=["mkdir", "-p", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s" %(ts[0], ts[1]))],
 718                         haltOnFailure = True
 719                 ))
 720
 721                 factory.addStep(ShellCommand(
 722                         name = "kmodprepare",
 723                         description = "Preparing kmod archive",
 724                         command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
 725                                  WithProperties("bin/targets/%s/%s%%(libc)s/packages/" %(ts[0], ts[1])),
 726                                  WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))],
 727                         haltOnFailure = True
 728                 ))
 729
 730                 factory.addStep(ShellCommand(
 731                         name = "kmodindex",
 732                         description = "Indexing kmod archive",
 733                         command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s",
 734                                  WithProperties("PACKAGE_SUBDIRS=bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))],
 735                         env = MakeEnv(),
 736                         haltOnFailure = True
 737                 ))
 738
 739                 # find rootfs staging directory
 740                 factory.addStep(SetProperty(
 741                         name = "stageroot",
 742                         property = "stageroot",
 743                         description = "Finding the rootfs staging directory",
 744                         command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"],
 745                         env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) }
 746                 ))
 747
 748                 factory.addStep(ShellCommand(
 749                         name = "filesdir",
 750                         description = "Creating file overlay directory",
 751                         command=["mkdir", "-p", "files/etc/opkg"],
 752                         haltOnFailure = True
 753                 ))
 754
 755                 factory.addStep(ShellCommand(
 756                         name = "kmodconfig",
 757                         description = "Embedding kmod repository configuration",
 758                         command=WithProperties("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(kernelversion)s#' " +
 759                                                "%(stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"),
 760                         haltOnFailure = True
 761                 ))
 762
 763         #factory.addStep(IfBuiltinShellCommand(
 764         factory.addStep(ShellCommand(
 765                 name = "images",
 766                 description = "Building and installing images",
 767                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
 768                 env = MakeEnv(),
 769                 haltOnFailure = True
 770         ))
 771
 772         factory.addStep(ShellCommand(
 773                 name = "diffconfig",
 774                 description = "Generating config.seed",
 775                 command=["make", "-j1", "diffconfig", "V=s"],
 776                 env = MakeEnv(),
 777                 haltOnFailure = True
 778         ))
 779
 780         factory.addStep(ShellCommand(
 781                 name = "checksums",
 782                 description = "Calculating checksums",
 783                 command=["make", "-j1", "checksum", "V=s"],
 784                 env = MakeEnv(),
 785                 haltOnFailure = True
 786         ))
 787
 788         # sign
 789         if gpg_keyid is not None:
 790                 factory.addStep(MasterShellCommand(
 791                         name = "signprepare",
 792                         description = "Preparing temporary signing directory",
 793                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 794                         haltOnFailure = True
 795                 ))
 796
 797                 factory.addStep(ShellCommand(
 798                         name = "signpack",
 799                         description = "Packing files to sign",
 800                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ bin/targets/%s/%s%%(libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1], ts[0], ts[1])),
 801                         haltOnFailure = True
 802                 ))
 803
 804                 factory.addStep(FileUpload(
 805                         slavesrc = "sign.tar.gz",
 806                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 807                         haltOnFailure = True
 808                 ))
 809
 810                 factory.addStep(MasterShellCommand(
 811                         name = "signfiles",
 812                         description = "Signing files",
 813                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 814                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 815                         haltOnFailure = True
 816                 ))
 817
 818                 factory.addStep(FileDownload(
 819                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 820                         slavedest = "sign.tar.gz",
 821                         haltOnFailure = True
 822                 ))
 823
 824                 factory.addStep(ShellCommand(
 825                         name = "signunpack",
 826                         description = "Unpacking signed files",
 827                         command = ["tar", "-xzf", "sign.tar.gz"],
 828                         haltOnFailure = True
 829                 ))
 830
 831         # upload
 832         factory.addStep(ShellCommand(
 833                 name = "dirprepare",
 834                 description = "Preparing upload directory structure",
 835                 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 836                 haltOnFailure = True
 837         ))
 838
 839         factory.addStep(ShellCommand(
 840                 name = "linkprepare",
 841                 description = "Preparing repository symlink",
 842                 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
 843                 doStepIf = IsNoMasterBuild,
 844                 haltOnFailure = True
 845         ))
 846
 847         if enable_kmod_archive:
 848                 factory.addStep(ShellCommand(
 849                         name = "kmoddirprepare",
 850                         description = "Preparing kmod archive upload directory",
 851                         command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 852                         haltOnFailure = True
 853                 ))
 854
 855         factory.addStep(ShellCommand(
 856                 name = "dirupload",
 857                 description = "Uploading directory structure",
 858                 command = ["rsync", "-4", "--info=name", "-az", "tmp/upload/", "%s/" %(rsync_bin_url)],
 859                 env={'RSYNC_PASSWORD': rsync_bin_key},
 860                 haltOnFailure = True,
 861                 logEnviron = False,
 862                 locks = NetLockUl,
 863         ))
 864
 865         factory.addStep(FileDownload(
 866                         mastersrc = "rsync.sh",
 867                         slavedest = "../rsync.sh",
 868                         mode = 0755))
 869
 870         factory.addStep(ShellCommand(
 871                 name = "targetupload",
 872                 description = "Uploading target files",
 873                 command=["../rsync.sh", "-4", "--info=name", "--exclude=/kmods/", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 874                          "-a", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
 875                          WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 876                 env={'RSYNC_PASSWORD': rsync_bin_key},
 877                 haltOnFailure = True,
 878                 logEnviron = False,
 879                 locks = NetLockUl,
 880         ))
 881
 882         if enable_kmod_archive:
 883                 factory.addStep(ShellCommand(
 884                         name = "kmodupload",
 885                         description = "Uploading kmod archive",
 886                         command=["../rsync.sh", "-4", "--info=name", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 887                                  "-a", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])),
 888                                  WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 889                         env={'RSYNC_PASSWORD': rsync_bin_key},
 890                         haltOnFailure = True,
 891                         logEnviron = False,
 892                         locks = NetLockUl,
 893                 ))
 894
 895         if rsync_src_url is not None:
 896                 factory.addStep(ShellCommand(
 897                         name = "sourceupload",
 898                         description = "Uploading source archives",
 899                         command=["../rsync.sh", "-4", "--info=name", "--size-only", "--delay-updates",
 900                                  WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-a", "dl/", "%s/" %(rsync_src_url)],
 901                         env={'RSYNC_PASSWORD': rsync_src_key},
 902                         haltOnFailure = True,
 903                         logEnviron = False,
 904                         locks = NetLockUl,
 905                 ))
 906
 907         if False:
 908                 factory.addStep(ShellCommand(
 909                         name = "packageupload",
 910                         description = "Uploading package files",
 911                         command=["../rsync.sh", "-4", "--info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 912                         env={'RSYNC_PASSWORD': rsync_bin_key},
 913                         haltOnFailure = False,
 914                         logEnviron = False,
 915                         locks = NetLockUl,
 916                 ))
 917
 918         # logs
 919         if False:
 920                 factory.addStep(ShellCommand(
 921                         name = "upload",
 922                         description = "Uploading logs",
 923                         command=["../rsync.sh", "-4", "-info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 924                         env={'RSYNC_PASSWORD': rsync_bin_key},
 925                         haltOnFailure = False,
 926                         alwaysRun = True,
 927                         logEnviron = False,
 928                         locks = NetLockUl,
 929                 ))
 930
 931         factory.addStep(ShellCommand(
 932                 name = "df",
 933                 description = "Reporting disk usage",
 934                 command=["df", "-h", "."],
 935                 env={'LC_ALL': 'C'},
 936                 haltOnFailure = False,
 937                 alwaysRun = True
 938         ))
 939
 940         from buildbot.config import BuilderConfig
 941
 942         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))
 943
 944
 945 ####### STATUS TARGETS
 946
 947 # 'status' is a list of Status Targets. The results of each build will be
 948 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 949 # including web pages, email senders, and IRC bots.
 950
 951 c['status'] = []
 952
 953 from buildbot.status import html
 954 from buildbot.status.web import authz, auth
 955
 956 if ini.has_option("status", "bind"):
 957         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 958                 authz_cfg=authz.Authz(
 959                         # change any of these to True to enable; see the manual for more
 960                         # options
 961                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 962                         gracefulShutdown = 'auth',
 963                         forceBuild = 'auth', # use this to test your slave once it is set up
 964                         forceAllBuilds = 'auth',
 965                         pingBuilder = False,
 966                         stopBuild = 'auth',
 967                         stopAllBuilds = 'auth',
 968                         cancelPendingBuild = 'auth',
 969                 )
 970                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 971         else:
 972                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 973
 974
 975 from buildbot.status import words
 976
 977 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 978         irc_host = ini.get("irc", "host")
 979         irc_port = 6667
 980         irc_chan = ini.get("irc", "channel")
 981         irc_nick = ini.get("irc", "nickname")
 982         irc_pass = None
 983
 984         if ini.has_option("irc", "port"):
 985                 irc_port = ini.getint("irc", "port")
 986
 987         if ini.has_option("irc", "password"):
 988                 irc_pass = ini.get("irc", "password")
 989
 990         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 991                         channels = [{ "channel": irc_chan }],
 992                         notify_events = {
 993                           'exception': 1,
 994                           'successToFailure': 1,
 995                           'failureToSuccess': 1
 996                         }
 997         )
 998
 999         c['status'].append(irc)
1000
1001 ####### DB URL
1002
1003 c['db'] = {
1004         # This specifies what database buildbot uses to store its state.  You can leave
1005         # this at its default for all but the largest installations.
1006         'db_url' : "sqlite:///state.sqlite",
1007 }